Collaborative-Filtering-RE-Pearson/Collaborative Filtering Recommendation Engine.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# This Notebook contains code for Building Collaborative Filtering based Recommendation Engine"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import boto3\n",
    "s3 = boto3.resource('s3')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "s3.Bucket('recommendationsystemshubham').download_file('dataset/movies.dat', 'datasets/movies.dat')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "s3.Bucket('recommendationsystemshubham').download_file('dataset/ratings.dat', 'datasets/ratings.dat')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "s3.Bucket('recommendationsystemshubham').download_file('dataset/users.dat', 'datasets/users.dat')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "DATASET_NAME = \"datasets/movies.csv\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages/ipykernel/__main__.py:4: ParserWarning: Falling back to the 'python' engine because the 'c' engine does not support regex separators (separators > 1 char and different from '\\s+' are interpreted as regex); you can avoid this warning by specifying engine='python'.\n"
     ]
    }
   ],
   "source": [
    "import pandas as pd\n",
    "\n",
    "unames = ['user_id', 'gender', 'age', 'occupation', 'zip']\n",
    "users = pd.read_table('datasets/users.dat', sep='::', header=None, names=unames)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages/ipykernel/__main__.py:2: ParserWarning: Falling back to the 'python' engine because the 'c' engine does not support regex separators (separators > 1 char and different from '\\s+' are interpreted as regex); you can avoid this warning by specifying engine='python'.\n",
      "  from ipykernel import kernelapp as app\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>user_id</th>\n",
       "      <th>movie_id</th>\n",
       "      <th>rating</th>\n",
       "      <th>timestamp</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>1193</td>\n",
       "      <td>5</td>\n",
       "      <td>978300760</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>661</td>\n",
       "      <td>3</td>\n",
       "      <td>978302109</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1</td>\n",
       "      <td>914</td>\n",
       "      <td>3</td>\n",
       "      <td>978301968</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1</td>\n",
       "      <td>3408</td>\n",
       "      <td>4</td>\n",
       "      <td>978300275</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1</td>\n",
       "      <td>2355</td>\n",
       "      <td>5</td>\n",
       "      <td>978824291</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   user_id  movie_id  rating  timestamp\n",
       "0        1      1193       5  978300760\n",
       "1        1       661       3  978302109\n",
       "2        1       914       3  978301968\n",
       "3        1      3408       4  978300275\n",
       "4        1      2355       5  978824291"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "rnames=['user_id', 'movie_id', 'rating', 'timestamp']\n",
    "rnames = pd.read_table('datasets/ratings.dat', sep='::', names = rnames, header=None)\n",
    "rnames[:5]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages/ipykernel/__main__.py:2: ParserWarning: Falling back to the 'python' engine because the 'c' engine does not support regex separators (separators > 1 char and different from '\\s+' are interpreted as regex); you can avoid this warning by specifying engine='python'.\n",
      "  from ipykernel import kernelapp as app\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>movie_id</th>\n",
       "      <th>title</th>\n",
       "      <th>genres</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>Toy Story (1995)</td>\n",
       "      <td>Animation|Children's|Comedy</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>Jumanji (1995)</td>\n",
       "      <td>Adventure|Children's|Fantasy</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3</td>\n",
       "      <td>Grumpier Old Men (1995)</td>\n",
       "      <td>Comedy|Romance</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>4</td>\n",
       "      <td>Waiting to Exhale (1995)</td>\n",
       "      <td>Comedy|Drama</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>5</td>\n",
       "      <td>Father of the Bride Part II (1995)</td>\n",
       "      <td>Comedy</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   movie_id                               title                        genres\n",
       "0         1                    Toy Story (1995)   Animation|Children's|Comedy\n",
       "1         2                      Jumanji (1995)  Adventure|Children's|Fantasy\n",
       "2         3             Grumpier Old Men (1995)                Comedy|Romance\n",
       "3         4            Waiting to Exhale (1995)                  Comedy|Drama\n",
       "4         5  Father of the Bride Part II (1995)                        Comedy"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "mnames = ['movie_id', 'title', 'genres']\n",
    "mnames = pd.read_table('datasets/movies.dat', sep='::', names = mnames, header=None)\n",
    "mnames[:5]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "movielens = pd.merge(pd.merge(rnames, users), mnames)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>user_id</th>\n",
       "      <th>movie_id</th>\n",
       "      <th>rating</th>\n",
       "      <th>timestamp</th>\n",
       "      <th>gender</th>\n",
       "      <th>age</th>\n",
       "      <th>occupation</th>\n",
       "      <th>zip</th>\n",
       "      <th>title</th>\n",
       "      <th>genres</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>1193</td>\n",
       "      <td>5</td>\n",
       "      <td>978300760</td>\n",
       "      <td>F</td>\n",
       "      <td>1</td>\n",
       "      <td>10</td>\n",
       "      <td>48067</td>\n",
       "      <td>One Flew Over the Cuckoo's Nest (1975)</td>\n",
       "      <td>Drama</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>1193</td>\n",
       "      <td>5</td>\n",
       "      <td>978298413</td>\n",
       "      <td>M</td>\n",
       "      <td>56</td>\n",
       "      <td>16</td>\n",
       "      <td>70072</td>\n",
       "      <td>One Flew Over the Cuckoo's Nest (1975)</td>\n",
       "      <td>Drama</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>12</td>\n",
       "      <td>1193</td>\n",
       "      <td>4</td>\n",
       "      <td>978220179</td>\n",
       "      <td>M</td>\n",
       "      <td>25</td>\n",
       "      <td>12</td>\n",
       "      <td>32793</td>\n",
       "      <td>One Flew Over the Cuckoo's Nest (1975)</td>\n",
       "      <td>Drama</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>15</td>\n",
       "      <td>1193</td>\n",
       "      <td>4</td>\n",
       "      <td>978199279</td>\n",
       "      <td>M</td>\n",
       "      <td>25</td>\n",
       "      <td>7</td>\n",
       "      <td>22903</td>\n",
       "      <td>One Flew Over the Cuckoo's Nest (1975)</td>\n",
       "      <td>Drama</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>17</td>\n",
       "      <td>1193</td>\n",
       "      <td>5</td>\n",
       "      <td>978158471</td>\n",
       "      <td>M</td>\n",
       "      <td>50</td>\n",
       "      <td>1</td>\n",
       "      <td>95350</td>\n",
       "      <td>One Flew Over the Cuckoo's Nest (1975)</td>\n",
       "      <td>Drama</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   user_id  movie_id  rating  timestamp gender  age  occupation    zip  \\\n",
       "0        1      1193       5  978300760      F    1          10  48067   \n",
       "1        2      1193       5  978298413      M   56          16  70072   \n",
       "2       12      1193       4  978220179      M   25          12  32793   \n",
       "3       15      1193       4  978199279      M   25           7  22903   \n",
       "4       17      1193       5  978158471      M   50           1  95350   \n",
       "\n",
       "                                    title genres  \n",
       "0  One Flew Over the Cuckoo's Nest (1975)  Drama  \n",
       "1  One Flew Over the Cuckoo's Nest (1975)  Drama  \n",
       "2  One Flew Over the Cuckoo's Nest (1975)  Drama  \n",
       "3  One Flew Over the Cuckoo's Nest (1975)  Drama  \n",
       "4  One Flew Over the Cuckoo's Nest (1975)  Drama  "
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "movielens[:5]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Following will be the procedure to perform collaborative filtering.\n",
    "\n",
    "### Performing non-personalized recommendation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "meanRatings = movielens.pivot_table('rating', index='title', columns='gender', aggfunc='mean')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th>gender</th>\n",
       "      <th>F</th>\n",
       "      <th>M</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>title</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>$1,000,000 Duck (1971)</th>\n",
       "      <td>3.375000</td>\n",
       "      <td>2.761905</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>'Night Mother (1986)</th>\n",
       "      <td>3.388889</td>\n",
       "      <td>3.352941</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>'Til There Was You (1997)</th>\n",
       "      <td>2.675676</td>\n",
       "      <td>2.733333</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>'burbs, The (1989)</th>\n",
       "      <td>2.793478</td>\n",
       "      <td>2.962085</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...And Justice for All (1979)</th>\n",
       "      <td>3.828571</td>\n",
       "      <td>3.689024</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "gender                                F         M\n",
       "title                                            \n",
       "$1,000,000 Duck (1971)         3.375000  2.761905\n",
       "'Night Mother (1986)           3.388889  3.352941\n",
       "'Til There Was You (1997)      2.675676  2.733333\n",
       "'burbs, The (1989)             2.793478  2.962085\n",
       "...And Justice for All (1979)  3.828571  3.689024"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "meanRatings[:5]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "ratings_by_title = movielens.groupby('title').size()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "moviesWithGreaterRatings = ratings_by_title.index[ratings_by_title > 200]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index([''burbs, The (1989)', '10 Things I Hate About You (1999)',\n",
       "       '101 Dalmatians (1961)', '101 Dalmatians (1996)', '12 Angry Men (1957)',\n",
       "       '13th Warrior, The (1999)', '2 Days in the Valley (1996)',\n",
       "       '20,000 Leagues Under the Sea (1954)', '2001: A Space Odyssey (1968)',\n",
       "       '2010 (1984)',\n",
       "       ...\n",
       "       'Year of Living Dangerously (1982)', 'Yellow Submarine (1968)',\n",
       "       'Yojimbo (1961)', 'You've Got Mail (1998)', 'Young Frankenstein (1974)',\n",
       "       'Young Guns (1988)', 'Young Guns II (1990)',\n",
       "       'Young Sherlock Holmes (1985)', 'Zero Effect (1998)',\n",
       "       'eXistenZ (1999)'],\n",
       "      dtype='object', name='title', length=1420)"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "moviesWithGreaterRatings"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "meanRatingsForSuchMovies = meanRatings.loc[moviesWithGreaterRatings,:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th>gender</th>\n",
       "      <th>F</th>\n",
       "      <th>M</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>title</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>'burbs, The (1989)</th>\n",
       "      <td>2.793478</td>\n",
       "      <td>2.962085</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10 Things I Hate About You (1999)</th>\n",
       "      <td>3.646552</td>\n",
       "      <td>3.311966</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>101 Dalmatians (1961)</th>\n",
       "      <td>3.791444</td>\n",
       "      <td>3.500000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>101 Dalmatians (1996)</th>\n",
       "      <td>3.240000</td>\n",
       "      <td>2.911215</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12 Angry Men (1957)</th>\n",
       "      <td>4.184397</td>\n",
       "      <td>4.328421</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13th Warrior, The (1999)</th>\n",
       "      <td>3.112000</td>\n",
       "      <td>3.168000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2 Days in the Valley (1996)</th>\n",
       "      <td>3.488889</td>\n",
       "      <td>3.244813</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20,000 Leagues Under the Sea (1954)</th>\n",
       "      <td>3.670103</td>\n",
       "      <td>3.709205</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2001: A Space Odyssey (1968)</th>\n",
       "      <td>3.825581</td>\n",
       "      <td>4.129738</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2010 (1984)</th>\n",
       "      <td>3.446809</td>\n",
       "      <td>3.413712</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "gender                                      F         M\n",
       "title                                                  \n",
       "'burbs, The (1989)                   2.793478  2.962085\n",
       "10 Things I Hate About You (1999)    3.646552  3.311966\n",
       "101 Dalmatians (1961)                3.791444  3.500000\n",
       "101 Dalmatians (1996)                3.240000  2.911215\n",
       "12 Angry Men (1957)                  4.184397  4.328421\n",
       "13th Warrior, The (1999)             3.112000  3.168000\n",
       "2 Days in the Valley (1996)          3.488889  3.244813\n",
       "20,000 Leagues Under the Sea (1954)  3.670103  3.709205\n",
       "2001: A Space Odyssey (1968)         3.825581  4.129738\n",
       "2010 (1984)                          3.446809  3.413712"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "meanRatingsForSuchMovies[:10]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th>gender</th>\n",
       "      <th>F</th>\n",
       "      <th>M</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>title</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>Clean Slate (Coup de Torchon) (1981)</th>\n",
       "      <td>5.0</td>\n",
       "      <td>3.857143</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Ballad of Narayama, The (Narayama Bushiko) (1958)</th>\n",
       "      <td>5.0</td>\n",
       "      <td>3.428571</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Raw Deal (1948)</th>\n",
       "      <td>5.0</td>\n",
       "      <td>3.307692</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Bittersweet Motel (2000)</th>\n",
       "      <td>5.0</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Skipped Parts (2000)</th>\n",
       "      <td>5.0</td>\n",
       "      <td>4.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Lamerica (1994)</th>\n",
       "      <td>5.0</td>\n",
       "      <td>4.666667</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Gambler, The (A J�t�kos) (1997)</th>\n",
       "      <td>5.0</td>\n",
       "      <td>3.166667</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Brother, Can You Spare a Dime? (1975)</th>\n",
       "      <td>5.0</td>\n",
       "      <td>3.642857</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Ayn Rand: A Sense of Life (1997)</th>\n",
       "      <td>5.0</td>\n",
       "      <td>4.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24 7: Twenty Four Seven (1997)</th>\n",
       "      <td>5.0</td>\n",
       "      <td>3.750000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "gender                                               F         M\n",
       "title                                                           \n",
       "Clean Slate (Coup de Torchon) (1981)               5.0  3.857143\n",
       "Ballad of Narayama, The (Narayama Bushiko) (1958)  5.0  3.428571\n",
       "Raw Deal (1948)                                    5.0  3.307692\n",
       "Bittersweet Motel (2000)                           5.0       NaN\n",
       "Skipped Parts (2000)                               5.0  4.000000\n",
       "Lamerica (1994)                                    5.0  4.666667\n",
       "Gambler, The (A J�t�kos) (1997)                    5.0  3.166667\n",
       "Brother, Can You Spare a Dime? (1975)              5.0  3.642857\n",
       "Ayn Rand: A Sense of Life (1997)                   5.0  4.000000\n",
       "24 7: Twenty Four Seven (1997)                     5.0  3.750000"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "top_female_ratings = meanRatings.sort_values(by='F', ascending=False)\n",
    "top_female_ratings[:10]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "rating_std = movielens.groupby('title')['rating'].std()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "title\n",
       "...And Justice for All (1979)    0.878110\n",
       "'Til There Was You (1997)        1.020159\n",
       "$1,000,000 Duck (1971)           1.092563\n",
       "'burbs, The (1989)               1.107760\n",
       "'Night Mother (1986)             1.118636\n",
       "Name: rating, dtype: float64"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "rating_std[:5].sort_values()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "rating_std = rating_std.loc[moviesWithGreaterRatings]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "title\n",
       "'burbs, The (1989)                                                    1.107760\n",
       "10 Things I Hate About You (1999)                                     0.989815\n",
       "101 Dalmatians (1961)                                                 0.982103\n",
       "101 Dalmatians (1996)                                                 1.098717\n",
       "12 Angry Men (1957)                                                   0.812731\n",
       "13th Warrior, The (1999)                                              1.140421\n",
       "2 Days in the Valley (1996)                                           0.921592\n",
       "20,000 Leagues Under the Sea (1954)                                   0.869685\n",
       "2001: A Space Odyssey (1968)                                          1.042504\n",
       "2010 (1984)                                                           0.946618\n",
       "28 Days (2000)                                                        0.920278\n",
       "39 Steps, The (1935)                                                  0.853501\n",
       "54 (1998)                                                             1.035481\n",
       "7th Voyage of Sinbad, The (1958)                                      0.931531\n",
       "8 1/2 (1963)                                                          0.946763\n",
       "8MM (1999)                                                            1.113432\n",
       "Abbott and Costello Meet Frankenstein (1948)                          1.074768\n",
       "About Last Night... (1986)                                            0.943025\n",
       "Absent Minded Professor, The (1961)                                   0.938965\n",
       "Absolute Power (1997)                                                 0.891659\n",
       "Abyss, The (1989)                                                     0.948102\n",
       "Ace Ventura: Pet Detective (1994)                                     1.140481\n",
       "Ace Ventura: When Nature Calls (1995)                                 1.178748\n",
       "Addams Family Values (1993)                                           1.013979\n",
       "Addams Family, The (1991)                                             0.974064\n",
       "Addicted to Love (1997)                                               0.931997\n",
       "Adventures in Babysitting (1987)                                      1.001816\n",
       "Adventures of Buckaroo Bonzai Across the 8th Dimension, The (1984)    1.161592\n",
       "Adventures of Priscilla, Queen of the Desert, The (1994)              1.009933\n",
       "Adventures of Robin Hood, The (1938)                                  0.912244\n",
       "                                                                        ...   \n",
       "Wild Things (1998)                                                    1.032785\n",
       "Wild Wild West (1999)                                                 1.065205\n",
       "William Shakespeare's Romeo and Juliet (1996)                         1.167051\n",
       "Willow (1988)                                                         1.002175\n",
       "Willy Wonka and the Chocolate Factory (1971)                          1.016170\n",
       "Wings of Desire (Der Himmel �ber Berlin) (1987)                       1.030626\n",
       "Wings of the Dove, The (1997)                                         0.868274\n",
       "Winnie the Pooh and the Blustery Day (1968)                           1.055202\n",
       "With Honors (1994)                                                    0.943410\n",
       "Witness (1985)                                                        0.795913\n",
       "Wizard of Oz, The (1939)                                              0.895978\n",
       "Wolf (1994)                                                           1.081220\n",
       "Women on the Verge of a Nervous Breakdown (1988)                      0.882129\n",
       "Wonder Boys (2000)                                                    0.880473\n",
       "Working Girl (1988)                                                   0.879409\n",
       "World Is Not Enough, The (1999)                                       0.972892\n",
       "Wrong Trousers, The (1993)                                            0.708666\n",
       "Wyatt Earp (1994)                                                     0.922257\n",
       "X-Files: Fight the Future, The (1998)                                 0.897204\n",
       "X-Men (2000)                                                          0.915067\n",
       "Year of Living Dangerously (1982)                                     0.880997\n",
       "Yellow Submarine (1968)                                               1.075429\n",
       "Yojimbo (1961)                                                        0.853238\n",
       "You've Got Mail (1998)                                                1.037854\n",
       "Young Frankenstein (1974)                                             0.872810\n",
       "Young Guns (1988)                                                     1.017437\n",
       "Young Guns II (1990)                                                  1.071959\n",
       "Young Sherlock Holmes (1985)                                          0.891176\n",
       "Zero Effect (1998)                                                    1.042932\n",
       "eXistenZ (1999)                                                       1.178568\n",
       "Name: rating, Length: 1420, dtype: float64"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "rating_std"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Evaluation Mechanism\n",
    "\n",
    "### -> taking smaller sample from the full dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(10000, 10)\n",
      "3707\n",
      "2279\n"
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "movielens = movielens.iloc[np.random.choice(movielens.index, size=10000, replace=False),:]\n",
    "print(movielens.shape)\n",
    "print(movielens.user_id.nunique())\n",
    "print(movielens.movie_id.nunique())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>user_id</th>\n",
       "      <th>movie_id</th>\n",
       "      <th>rating</th>\n",
       "      <th>timestamp</th>\n",
       "      <th>gender</th>\n",
       "      <th>age</th>\n",
       "      <th>occupation</th>\n",
       "      <th>zip</th>\n",
       "      <th>title</th>\n",
       "      <th>genres</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>562119</th>\n",
       "      <td>3485</td>\n",
       "      <td>1288</td>\n",
       "      <td>5</td>\n",
       "      <td>967094696</td>\n",
       "      <td>M</td>\n",
       "      <td>25</td>\n",
       "      <td>0</td>\n",
       "      <td>94121</td>\n",
       "      <td>This Is Spinal Tap (1984)</td>\n",
       "      <td>Comedy|Drama|Musical</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>133604</th>\n",
       "      <td>475</td>\n",
       "      <td>1198</td>\n",
       "      <td>5</td>\n",
       "      <td>976218884</td>\n",
       "      <td>F</td>\n",
       "      <td>25</td>\n",
       "      <td>2</td>\n",
       "      <td>55421</td>\n",
       "      <td>Raiders of the Lost Ark (1981)</td>\n",
       "      <td>Action|Adventure</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>755282</th>\n",
       "      <td>4865</td>\n",
       "      <td>2132</td>\n",
       "      <td>5</td>\n",
       "      <td>962818748</td>\n",
       "      <td>F</td>\n",
       "      <td>35</td>\n",
       "      <td>14</td>\n",
       "      <td>02130</td>\n",
       "      <td>Who's Afraid of Virginia Woolf? (1966)</td>\n",
       "      <td>Drama</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>539532</th>\n",
       "      <td>3642</td>\n",
       "      <td>1220</td>\n",
       "      <td>3</td>\n",
       "      <td>966473693</td>\n",
       "      <td>F</td>\n",
       "      <td>25</td>\n",
       "      <td>7</td>\n",
       "      <td>98188</td>\n",
       "      <td>Blues Brothers, The (1980)</td>\n",
       "      <td>Action|Comedy|Musical</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>922842</th>\n",
       "      <td>6036</td>\n",
       "      <td>2672</td>\n",
       "      <td>1</td>\n",
       "      <td>956710231</td>\n",
       "      <td>F</td>\n",
       "      <td>25</td>\n",
       "      <td>15</td>\n",
       "      <td>32603</td>\n",
       "      <td>Thirteenth Floor, The (1999)</td>\n",
       "      <td>Drama|Sci-Fi|Thriller</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        user_id  movie_id  rating  timestamp gender  age  occupation    zip  \\\n",
       "562119     3485      1288       5  967094696      M   25           0  94121   \n",
       "133604      475      1198       5  976218884      F   25           2  55421   \n",
       "755282     4865      2132       5  962818748      F   35          14  02130   \n",
       "539532     3642      1220       3  966473693      F   25           7  98188   \n",
       "922842     6036      2672       1  956710231      F   25          15  32603   \n",
       "\n",
       "                                         title                 genres  \n",
       "562119               This Is Spinal Tap (1984)   Comedy|Drama|Musical  \n",
       "133604          Raiders of the Lost Ark (1981)       Action|Adventure  \n",
       "755282  Who's Afraid of Virginia Woolf? (1966)                  Drama  \n",
       "539532              Blues Brothers, The (1980)  Action|Comedy|Musical  \n",
       "922842            Thirteenth Floor, The (1999)  Drama|Sci-Fi|Thriller  "
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "movielens.iloc[:5,: ]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "condition = lambda x: len(x.index) > 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "movielens = movielens.groupby('user_id').filter(condition)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(8443, 10)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "print (movielens.shape)\n",
    "np.all(movielens.user_id.value_counts() > 1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Generating train and test subsets of dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(5797, 11)\n",
      "(2646, 11)\n",
      "Int64Index([], dtype='int64')\n"
     ]
    }
   ],
   "source": [
    "def assign_to_set(df):\n",
    "    sampled_ids = np.random.choice(df.index,\n",
    "                                   size=np.int64(np.ceil(df.index.size * 0.2)),\n",
    "                                   replace=False)\n",
    "    df.loc[sampled_ids, 'for_testing'] = True\n",
    "    return df\n",
    "\n",
    "movielens['for_testing'] = False\n",
    "grouped = movielens.groupby('user_id', group_keys=False).apply(assign_to_set)\n",
    "movielens_train = movielens[grouped.for_testing == False]\n",
    "movielens_test = movielens[grouped.for_testing == True]\n",
    "print (movielens_train.shape)\n",
    "print (movielens_test.shape)\n",
    "print (movielens_train.index & movielens_test.index)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>user_id</th>\n",
       "      <th>movie_id</th>\n",
       "      <th>rating</th>\n",
       "      <th>timestamp</th>\n",
       "      <th>gender</th>\n",
       "      <th>age</th>\n",
       "      <th>occupation</th>\n",
       "      <th>zip</th>\n",
       "      <th>title</th>\n",
       "      <th>genres</th>\n",
       "      <th>for_testing</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>133604</th>\n",
       "      <td>475</td>\n",
       "      <td>1198</td>\n",
       "      <td>5</td>\n",
       "      <td>976218884</td>\n",
       "      <td>F</td>\n",
       "      <td>25</td>\n",
       "      <td>2</td>\n",
       "      <td>55421</td>\n",
       "      <td>Raiders of the Lost Ark (1981)</td>\n",
       "      <td>Action|Adventure</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>755282</th>\n",
       "      <td>4865</td>\n",
       "      <td>2132</td>\n",
       "      <td>5</td>\n",
       "      <td>962818748</td>\n",
       "      <td>F</td>\n",
       "      <td>35</td>\n",
       "      <td>14</td>\n",
       "      <td>02130</td>\n",
       "      <td>Who's Afraid of Virginia Woolf? (1966)</td>\n",
       "      <td>Drama</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>539532</th>\n",
       "      <td>3642</td>\n",
       "      <td>1220</td>\n",
       "      <td>3</td>\n",
       "      <td>966473693</td>\n",
       "      <td>F</td>\n",
       "      <td>25</td>\n",
       "      <td>7</td>\n",
       "      <td>98188</td>\n",
       "      <td>Blues Brothers, The (1980)</td>\n",
       "      <td>Action|Comedy|Musical</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>922842</th>\n",
       "      <td>6036</td>\n",
       "      <td>2672</td>\n",
       "      <td>1</td>\n",
       "      <td>956710231</td>\n",
       "      <td>F</td>\n",
       "      <td>25</td>\n",
       "      <td>15</td>\n",
       "      <td>32603</td>\n",
       "      <td>Thirteenth Floor, The (1999)</td>\n",
       "      <td>Drama|Sci-Fi|Thriller</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>685094</th>\n",
       "      <td>4682</td>\n",
       "      <td>1094</td>\n",
       "      <td>4</td>\n",
       "      <td>964566489</td>\n",
       "      <td>M</td>\n",
       "      <td>25</td>\n",
       "      <td>7</td>\n",
       "      <td>05346</td>\n",
       "      <td>Crying Game, The (1992)</td>\n",
       "      <td>Drama|Romance|War</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        user_id  movie_id  rating  timestamp gender  age  occupation    zip  \\\n",
       "133604      475      1198       5  976218884      F   25           2  55421   \n",
       "755282     4865      2132       5  962818748      F   35          14  02130   \n",
       "539532     3642      1220       3  966473693      F   25           7  98188   \n",
       "922842     6036      2672       1  956710231      F   25          15  32603   \n",
       "685094     4682      1094       4  964566489      M   25           7  05346   \n",
       "\n",
       "                                         title                 genres  \\\n",
       "133604          Raiders of the Lost Ark (1981)       Action|Adventure   \n",
       "755282  Who's Afraid of Virginia Woolf? (1966)                  Drama   \n",
       "539532              Blues Brothers, The (1980)  Action|Comedy|Musical   \n",
       "922842            Thirteenth Floor, The (1999)  Drama|Sci-Fi|Thriller   \n",
       "685094                 Crying Game, The (1992)      Drama|Romance|War   \n",
       "\n",
       "        for_testing  \n",
       "133604        False  \n",
       "755282        False  \n",
       "539532        False  \n",
       "922842        False  \n",
       "685094        False  "
      ]
     },
     "execution_count": 40,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "movielens_train[:5]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [],
   "source": [
    "movielens_train.to_csv('datasets/movielens_train.csv')\n",
    "movielens_test.to_csv('datasets/movielens_test.csv')\n",
    "movielens.to_csv('datasets/movielens.csv')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Evaluation Model\n",
    "\n",
    "#### RMSE\n",
    "#### Precision/ Recall/ F-score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [],
   "source": [
    "def compute_rmse(y_pred, y_true):\n",
    "    \"\"\" Compute Root Mean Squared Error. \"\"\"\n",
    "    return np.sqrt(np.mean(np.power(y_pred - y_true, 2)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [],
   "source": [
    "def evaluate(estimate_f):\n",
    "    \"\"\" RMSE-based predictive performance evaluation with pandas. \"\"\"\n",
    "    ids_to_estimate = zip(movielens_test.user_id, movielens_test.movie_id)\n",
    "    estimated = np.array([estimate_f(u,i) for (u,i) in ids_to_estimate])\n",
    "    real = movielens_test.rating.values\n",
    "    return compute_rmse(estimated, real)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "RMSE for estimate1: 1.227093899694165\n"
     ]
    }
   ],
   "source": [
    "def estimate1(user_id, item_id):\n",
    "    \"\"\" Simple content-filtering based on mean ratings. \"\"\"\n",
    "    return movielens_train.loc[movielens_train.user_id == user_id, 'rating'].mean()\n",
    "\n",
    "print ('RMSE for estimate1: %s' % evaluate(estimate1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "RMSE for estimate2: 1.1253049972938671\n"
     ]
    }
   ],
   "source": [
    "def estimate2(user_id, movie_id):\n",
    "    \"\"\" Simple collaborative filter based on mean ratings. \"\"\"\n",
    "    ratings_by_others = movielens_train[movielens_train.movie_id == movie_id]\n",
    "    if ratings_by_others.empty: return 3.0\n",
    "    return ratings_by_others.rating.mean()\n",
    "\n",
    "print ('RMSE for estimate2: %s' % evaluate(estimate2))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## The above RMSE is cool but can we do better? let's try."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Let's do pivoting."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [],
   "source": [
    "ratings_mtx_df = movielens_train.pivot_table(values='rating',\n",
    "                                             index='user_id',\n",
    "                                             columns='movie_id')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th>movie_id</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "      <th>5</th>\n",
       "      <th>6</th>\n",
       "      <th>7</th>\n",
       "      <th>9</th>\n",
       "      <th>10</th>\n",
       "      <th>11</th>\n",
       "      <th>12</th>\n",
       "      <th>14</th>\n",
       "      <th>19</th>\n",
       "      <th>21</th>\n",
       "      <th>22</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>user_id</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>6000</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6002</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6003</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6007</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6010</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6011</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6016</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6022</th>\n",
       "      <td>5.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6023</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6030</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6033</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6035</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6036</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6037</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6040</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "movie_id   1   2   3   4   5   6   7   9   10  11  12  14  19  21  22\n",
       "user_id                                                              \n",
       "6000      NaN NaN NaN NaN NaN NaN NaN NaN NaN NaN NaN NaN NaN NaN NaN\n",
       "6002      NaN NaN NaN NaN NaN NaN NaN NaN NaN NaN NaN NaN NaN NaN NaN\n",
       "6003      NaN NaN NaN NaN NaN NaN NaN NaN NaN NaN NaN NaN NaN NaN NaN\n",
       "6007      NaN NaN NaN NaN NaN NaN NaN NaN NaN NaN NaN NaN NaN NaN NaN\n",
       "6010      NaN NaN NaN NaN NaN NaN NaN NaN NaN NaN NaN NaN NaN NaN NaN\n",
       "6011      NaN NaN NaN NaN NaN NaN NaN NaN NaN NaN NaN NaN NaN NaN NaN\n",
       "6016      NaN NaN NaN NaN NaN NaN NaN NaN NaN NaN NaN NaN NaN NaN NaN\n",
       "6022      5.0 NaN NaN NaN NaN NaN NaN NaN NaN NaN NaN NaN NaN NaN NaN\n",
       "6023      NaN NaN NaN NaN NaN NaN NaN NaN NaN NaN NaN NaN NaN NaN NaN\n",
       "6030      NaN NaN NaN NaN NaN NaN NaN NaN NaN NaN NaN NaN NaN NaN NaN\n",
       "6033      NaN NaN NaN NaN NaN NaN NaN NaN NaN NaN NaN NaN NaN NaN NaN\n",
       "6035      NaN NaN NaN NaN NaN NaN NaN NaN NaN NaN NaN NaN NaN NaN NaN\n",
       "6036      NaN NaN NaN NaN NaN NaN NaN NaN NaN NaN NaN NaN NaN NaN NaN\n",
       "6037      NaN NaN NaN NaN NaN NaN NaN NaN NaN NaN NaN NaN NaN NaN NaN\n",
       "6040      NaN NaN NaN NaN NaN NaN NaN NaN NaN NaN NaN NaN NaN NaN NaN"
      ]
     },
     "execution_count": 57,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ratings_mtx_df.iloc[-15:, :15]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [],
   "source": [
    "def pearson(s1, s2):\n",
    "    \"\"\"Take two pd.Series objects and return a pearson correlation.\"\"\"\n",
    "    s1_c = s1 - s1.mean()\n",
    "    s2_c = s2 - s2.mean()\n",
    "    return np.sum(s1_c * s2_c) / np.sqrt(np.sum(s1_c ** 2) * np.sum(s2_c ** 2))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {},
   "outputs": [],
   "source": [
    "class CollabFiltering:\n",
    "    \"\"\" Collaborative filtering using a custom sim(u,u'). \"\"\"\n",
    "\n",
    "    def learn(self):\n",
    "        \"\"\" Prepare datastructures for estimation. \"\"\"\n",
    "        self.all_user_profiles = movielens.pivot_table('rating', index='movie_id', columns='user_id')\n",
    "\n",
    "    def estimate(self, user_id, movie_id):\n",
    "        \"\"\" Ratings weighted by correlation similarity. \"\"\"\n",
    "        ratings_by_others = movielens_train[movielens_train.movie_id == movie_id]\n",
    "        if ratings_by_others.empty: return 3.0\n",
    "        ratings_by_others.set_index('user_id', inplace=True)\n",
    "        their_ids = ratings_by_others.index\n",
    "        their_ratings = ratings_by_others.rating\n",
    "        their_profiles = self.all_user_profiles[their_ids]\n",
    "        user_profile = self.all_user_profiles[user_id]\n",
    "        sims = their_profiles.apply(lambda profile: pearson(profile, user_profile), axis=0)\n",
    "        ratings_sims = pd.DataFrame({'sim': sims, 'rating': their_ratings})\n",
    "        ratings_sims = ratings_sims[ ratings_sims.sim > 0]\n",
    "        if ratings_sims.empty:\n",
    "            return their_ratings.mean()\n",
    "        else:\n",
    "            return np.average(ratings_sims.rating, weights=ratings_sims.sim)\n",
    "        \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages/ipykernel/__main__.py:5: RuntimeWarning: invalid value encountered in double_scalars\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "RMSE for CollabFiltering: 1.0783979099323924\n"
     ]
    }
   ],
   "source": [
    "reco = CollabFiltering()\n",
    "reco.learn()\n",
    "print ('RMSE for CollabFiltering: %s' % evaluate(reco.estimate) )"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## This was the collaborative Filtering based Recommendation system. We can further optimize this using genres as features."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "conda_python3",
   "language": "python",
   "name": "conda_python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}