code/matrix_factorisation_cross_validation.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Code preliminaries"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import  os\n",
    "\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "\n",
    "\n",
    "from create_df_larger import read_df_larger\n",
    "df, dfc, all_homes, appliance_min, national_average = read_df_larger()\n",
    "\n",
    "df = df.rename(columns={'house_num_rooms':'num_rooms',\n",
    "                        'num_occupants':'total_occupants',\n",
    "                        'difference_ratio_min_max':'ratio_difference_min_max'})\n",
    "K_min, K_max = 1,6\n",
    "F_min, F_max=1,8\n",
    "\n",
    "from all_functions import *\n",
    "from features_larger import *"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "appliance=\"hvac\"\n",
    "month=6\n",
    "all_homes = df['%s_%d' %(appliance, month)].dropna().index"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Selecting a small subset of the data."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "subset = df.head(15)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "subset_hvac = subset[['%s_%d' %(appliance, month) for month in range(5, 11)]].dropna()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Considering only the HVAC consumption from these few homes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>hvac_5</th>\n",
       "      <th>hvac_6</th>\n",
       "      <th>hvac_7</th>\n",
       "      <th>hvac_8</th>\n",
       "      <th>hvac_9</th>\n",
       "      <th>hvac_10</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>26</th>\n",
       "      <td>242.395737</td>\n",
       "      <td>543.781433</td>\n",
       "      <td>761.394043</td>\n",
       "      <td>902.418823</td>\n",
       "      <td>610.035828</td>\n",
       "      <td>395.169098</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>59</th>\n",
       "      <td>77.613205</td>\n",
       "      <td>197.951508</td>\n",
       "      <td>457.950378</td>\n",
       "      <td>700.443054</td>\n",
       "      <td>244.340973</td>\n",
       "      <td>81.064682</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>94</th>\n",
       "      <td>19.836948</td>\n",
       "      <td>66.621819</td>\n",
       "      <td>126.121681</td>\n",
       "      <td>155.045364</td>\n",
       "      <td>92.979385</td>\n",
       "      <td>33.392750</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>101</th>\n",
       "      <td>191.965454</td>\n",
       "      <td>325.858032</td>\n",
       "      <td>462.708923</td>\n",
       "      <td>485.186005</td>\n",
       "      <td>350.491455</td>\n",
       "      <td>192.567566</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>222</th>\n",
       "      <td>478.951691</td>\n",
       "      <td>896.467285</td>\n",
       "      <td>1077.890137</td>\n",
       "      <td>1139.740479</td>\n",
       "      <td>887.044739</td>\n",
       "      <td>584.328979</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>364</th>\n",
       "      <td>236.925064</td>\n",
       "      <td>535.010376</td>\n",
       "      <td>736.468933</td>\n",
       "      <td>850.080811</td>\n",
       "      <td>642.858459</td>\n",
       "      <td>378.799713</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>545</th>\n",
       "      <td>350.414551</td>\n",
       "      <td>778.204224</td>\n",
       "      <td>1065.381104</td>\n",
       "      <td>1120.907593</td>\n",
       "      <td>788.107544</td>\n",
       "      <td>375.303040</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>624</th>\n",
       "      <td>72.737854</td>\n",
       "      <td>191.232803</td>\n",
       "      <td>307.978149</td>\n",
       "      <td>389.598602</td>\n",
       "      <td>239.580887</td>\n",
       "      <td>22.099518</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>661</th>\n",
       "      <td>221.594315</td>\n",
       "      <td>648.794067</td>\n",
       "      <td>942.087769</td>\n",
       "      <td>1069.414429</td>\n",
       "      <td>756.477905</td>\n",
       "      <td>399.598877</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>871</th>\n",
       "      <td>84.128731</td>\n",
       "      <td>231.698425</td>\n",
       "      <td>403.408813</td>\n",
       "      <td>514.838928</td>\n",
       "      <td>319.413025</td>\n",
       "      <td>140.866745</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>946</th>\n",
       "      <td>217.752716</td>\n",
       "      <td>446.865326</td>\n",
       "      <td>605.236755</td>\n",
       "      <td>683.144165</td>\n",
       "      <td>452.278839</td>\n",
       "      <td>248.735733</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1169</th>\n",
       "      <td>21.098000</td>\n",
       "      <td>177.999435</td>\n",
       "      <td>327.977417</td>\n",
       "      <td>445.256683</td>\n",
       "      <td>281.603027</td>\n",
       "      <td>125.249001</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1283</th>\n",
       "      <td>512.822205</td>\n",
       "      <td>526.241333</td>\n",
       "      <td>1300.884033</td>\n",
       "      <td>1507.603027</td>\n",
       "      <td>1297.922485</td>\n",
       "      <td>452.018646</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1310</th>\n",
       "      <td>232.854477</td>\n",
       "      <td>300.484253</td>\n",
       "      <td>338.782837</td>\n",
       "      <td>366.564392</td>\n",
       "      <td>321.254883</td>\n",
       "      <td>273.744385</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "          hvac_5      hvac_6       hvac_7       hvac_8       hvac_9  \\\n",
       "26    242.395737  543.781433   761.394043   902.418823   610.035828   \n",
       "59     77.613205  197.951508   457.950378   700.443054   244.340973   \n",
       "94     19.836948   66.621819   126.121681   155.045364    92.979385   \n",
       "101   191.965454  325.858032   462.708923   485.186005   350.491455   \n",
       "222   478.951691  896.467285  1077.890137  1139.740479   887.044739   \n",
       "364   236.925064  535.010376   736.468933   850.080811   642.858459   \n",
       "545   350.414551  778.204224  1065.381104  1120.907593   788.107544   \n",
       "624    72.737854  191.232803   307.978149   389.598602   239.580887   \n",
       "661   221.594315  648.794067   942.087769  1069.414429   756.477905   \n",
       "871    84.128731  231.698425   403.408813   514.838928   319.413025   \n",
       "946   217.752716  446.865326   605.236755   683.144165   452.278839   \n",
       "1169   21.098000  177.999435   327.977417   445.256683   281.603027   \n",
       "1283  512.822205  526.241333  1300.884033  1507.603027  1297.922485   \n",
       "1310  232.854477  300.484253   338.782837   366.564392   321.254883   \n",
       "\n",
       "         hvac_10  \n",
       "26    395.169098  \n",
       "59     81.064682  \n",
       "94     33.392750  \n",
       "101   192.567566  \n",
       "222   584.328979  \n",
       "364   378.799713  \n",
       "545   375.303040  \n",
       "624    22.099518  \n",
       "661   399.598877  \n",
       "871   140.866745  \n",
       "946   248.735733  \n",
       "1169  125.249001  \n",
       "1283  452.018646  \n",
       "1310  273.744385  "
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "subset_hvac"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Description of the various attributes. Columns represent energy usage in months 5 through 10."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>hvac_5</th>\n",
       "      <th>hvac_6</th>\n",
       "      <th>hvac_7</th>\n",
       "      <th>hvac_8</th>\n",
       "      <th>hvac_9</th>\n",
       "      <th>hvac_10</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>14.000000</td>\n",
       "      <td>14.000000</td>\n",
       "      <td>14.000000</td>\n",
       "      <td>14.000000</td>\n",
       "      <td>14.000000</td>\n",
       "      <td>14.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>211.506496</td>\n",
       "      <td>419.086451</td>\n",
       "      <td>636.733641</td>\n",
       "      <td>737.874454</td>\n",
       "      <td>520.313531</td>\n",
       "      <td>264.495624</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std</th>\n",
       "      <td>154.569877</td>\n",
       "      <td>246.811850</td>\n",
       "      <td>351.783626</td>\n",
       "      <td>377.517354</td>\n",
       "      <td>327.900609</td>\n",
       "      <td>171.895097</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>min</th>\n",
       "      <td>19.836948</td>\n",
       "      <td>66.621819</td>\n",
       "      <td>126.121681</td>\n",
       "      <td>155.045364</td>\n",
       "      <td>92.979385</td>\n",
       "      <td>22.099518</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25%</th>\n",
       "      <td>79.242086</td>\n",
       "      <td>206.388237</td>\n",
       "      <td>354.939331</td>\n",
       "      <td>455.239014</td>\n",
       "      <td>291.055527</td>\n",
       "      <td>129.153437</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50%</th>\n",
       "      <td>219.673515</td>\n",
       "      <td>386.361679</td>\n",
       "      <td>533.972839</td>\n",
       "      <td>691.793610</td>\n",
       "      <td>401.385147</td>\n",
       "      <td>261.240059</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75%</th>\n",
       "      <td>241.028069</td>\n",
       "      <td>541.588669</td>\n",
       "      <td>896.914337</td>\n",
       "      <td>1027.665527</td>\n",
       "      <td>728.073044</td>\n",
       "      <td>391.076752</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max</th>\n",
       "      <td>512.822205</td>\n",
       "      <td>896.467285</td>\n",
       "      <td>1300.884033</td>\n",
       "      <td>1507.603027</td>\n",
       "      <td>1297.922485</td>\n",
       "      <td>584.328979</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "           hvac_5      hvac_6       hvac_7       hvac_8       hvac_9  \\\n",
       "count   14.000000   14.000000    14.000000    14.000000    14.000000   \n",
       "mean   211.506496  419.086451   636.733641   737.874454   520.313531   \n",
       "std    154.569877  246.811850   351.783626   377.517354   327.900609   \n",
       "min     19.836948   66.621819   126.121681   155.045364    92.979385   \n",
       "25%     79.242086  206.388237   354.939331   455.239014   291.055527   \n",
       "50%    219.673515  386.361679   533.972839   691.793610   401.385147   \n",
       "75%    241.028069  541.588669   896.914337  1027.665527   728.073044   \n",
       "max    512.822205  896.467285  1300.884033  1507.603027  1297.922485   \n",
       "\n",
       "          hvac_10  \n",
       "count   14.000000  \n",
       "mean   264.495624  \n",
       "std    171.895097  \n",
       "min     22.099518  \n",
       "25%    129.153437  \n",
       "50%    261.240059  \n",
       "75%    391.076752  \n",
       "max    584.328979  "
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "subset_hvac.describe()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Now, making one entry missing (Home 26, month #7) by putting NaN. True energy for this <home, appliance> is 761 units."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "subset_hvac.loc[26, 'hvac_7']=np.NAN"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "Normalising each column to be on the range (0, 1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "col_max = subset_hvac.max()\n",
    "col_min = subset_hvac.min()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "hvac_5      512.822205\n",
       "hvac_6      896.467285\n",
       "hvac_7     1300.884033\n",
       "hvac_8     1507.603027\n",
       "hvac_9     1297.922485\n",
       "hvac_10     584.328979\n",
       "dtype: float64"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "col_max"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Just checking if the normalisation is working correctly or not"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "26      0.451451\n",
       "59      0.117197\n",
       "94      0.000000\n",
       "101     0.349155\n",
       "222     0.931295\n",
       "364     0.440354\n",
       "545     0.670563\n",
       "624     0.107307\n",
       "661     0.409256\n",
       "871     0.130413\n",
       "946     0.401464\n",
       "1169    0.002558\n",
       "1283    1.000000\n",
       "1310    0.432097\n",
       "Name: hvac_5, dtype: float64"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "(subset_hvac['hvac_5']-col_min['hvac_5'])/(col_max['hvac_5']-col_min['hvac_5'])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Seems to be working great. All the numbers are in the range(0, 1). Now, doing this for the entire matrix."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "subset_hvac_normalised = subset_hvac.copy()\n",
    "for col in subset_hvac.columns:\n",
    "    subset_hvac_normalised[col] = (subset_hvac[col]-col_min[col])/(col_max[col]-col_min[col])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>hvac_5</th>\n",
       "      <th>hvac_6</th>\n",
       "      <th>hvac_7</th>\n",
       "      <th>hvac_8</th>\n",
       "      <th>hvac_9</th>\n",
       "      <th>hvac_10</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>26</th>\n",
       "      <td>0.451451</td>\n",
       "      <td>0.574998</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.552563</td>\n",
       "      <td>0.429113</td>\n",
       "      <td>0.663554</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>59</th>\n",
       "      <td>0.117197</td>\n",
       "      <td>0.158258</td>\n",
       "      <td>0.282465</td>\n",
       "      <td>0.403234</td>\n",
       "      <td>0.125617</td>\n",
       "      <td>0.104877</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>94</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.020087</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>101</th>\n",
       "      <td>0.349155</td>\n",
       "      <td>0.312391</td>\n",
       "      <td>0.286515</td>\n",
       "      <td>0.244086</td>\n",
       "      <td>0.213713</td>\n",
       "      <td>0.303200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>222</th>\n",
       "      <td>0.931295</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.810180</td>\n",
       "      <td>0.728024</td>\n",
       "      <td>0.659007</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>364</th>\n",
       "      <td>0.440354</td>\n",
       "      <td>0.564429</td>\n",
       "      <td>0.519550</td>\n",
       "      <td>0.513868</td>\n",
       "      <td>0.456353</td>\n",
       "      <td>0.634439</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>545</th>\n",
       "      <td>0.670563</td>\n",
       "      <td>0.857488</td>\n",
       "      <td>0.799531</td>\n",
       "      <td>0.714101</td>\n",
       "      <td>0.576897</td>\n",
       "      <td>0.628220</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>624</th>\n",
       "      <td>0.107307</td>\n",
       "      <td>0.150162</td>\n",
       "      <td>0.154803</td>\n",
       "      <td>0.173415</td>\n",
       "      <td>0.121667</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>661</th>\n",
       "      <td>0.409256</td>\n",
       "      <td>0.701543</td>\n",
       "      <td>0.694580</td>\n",
       "      <td>0.676030</td>\n",
       "      <td>0.550647</td>\n",
       "      <td>0.671433</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>871</th>\n",
       "      <td>0.130413</td>\n",
       "      <td>0.198925</td>\n",
       "      <td>0.236037</td>\n",
       "      <td>0.266010</td>\n",
       "      <td>0.187921</td>\n",
       "      <td>0.211243</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>946</th>\n",
       "      <td>0.401464</td>\n",
       "      <td>0.458210</td>\n",
       "      <td>0.407840</td>\n",
       "      <td>0.390445</td>\n",
       "      <td>0.298188</td>\n",
       "      <td>0.403103</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1169</th>\n",
       "      <td>0.002558</td>\n",
       "      <td>0.134215</td>\n",
       "      <td>0.171827</td>\n",
       "      <td>0.214565</td>\n",
       "      <td>0.156542</td>\n",
       "      <td>0.183465</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1283</th>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.553862</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.764668</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1310</th>\n",
       "      <td>0.432097</td>\n",
       "      <td>0.281814</td>\n",
       "      <td>0.181025</td>\n",
       "      <td>0.156384</td>\n",
       "      <td>0.189449</td>\n",
       "      <td>0.447584</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        hvac_5    hvac_6    hvac_7    hvac_8    hvac_9   hvac_10\n",
       "26    0.451451  0.574998       NaN  0.552563  0.429113  0.663554\n",
       "59    0.117197  0.158258  0.282465  0.403234  0.125617  0.104877\n",
       "94    0.000000  0.000000  0.000000  0.000000  0.000000  0.020087\n",
       "101   0.349155  0.312391  0.286515  0.244086  0.213713  0.303200\n",
       "222   0.931295  1.000000  0.810180  0.728024  0.659007  1.000000\n",
       "364   0.440354  0.564429  0.519550  0.513868  0.456353  0.634439\n",
       "545   0.670563  0.857488  0.799531  0.714101  0.576897  0.628220\n",
       "624   0.107307  0.150162  0.154803  0.173415  0.121667  0.000000\n",
       "661   0.409256  0.701543  0.694580  0.676030  0.550647  0.671433\n",
       "871   0.130413  0.198925  0.236037  0.266010  0.187921  0.211243\n",
       "946   0.401464  0.458210  0.407840  0.390445  0.298188  0.403103\n",
       "1169  0.002558  0.134215  0.171827  0.214565  0.156542  0.183465\n",
       "1283  1.000000  0.553862  1.000000  1.000000  1.000000  0.764668\n",
       "1310  0.432097  0.281814  0.181025  0.156384  0.189449  0.447584"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "subset_hvac_normalised"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Employing matrix factorisation now"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import numpy\n",
    "numpy.random.seed(0)\n",
    "def matrix_factorization(R, P, Q, K, steps=5000, alpha=0.0002, beta=0.02):\n",
    "    Q = Q.T\n",
    "    for step in xrange(steps):\n",
    "        for i in xrange(len(R)):\n",
    "            for j in xrange(len(R[i])):\n",
    "                if R[i][j]>0:\n",
    "                    eij = R[i][j] - numpy.dot(P[i,:],Q[:,j])\n",
    "                    for k in xrange(K):\n",
    "                        P[i][k] = P[i][k] + alpha * (2 * eij * Q[k][j] - beta * P[i][k])\n",
    "                        Q[k][j] = Q[k][j] + alpha * (2 * eij * P[i][k] - beta * Q[k][j])\n",
    "        eR = numpy.dot(P,Q)\n",
    "        e = 0\n",
    "        for i in xrange(len(R)):\n",
    "            for j in xrange(len(R[i])):\n",
    "                if R[i][j] > 0:\n",
    "                    e = e + pow(R[i][j] - numpy.dot(P[i,:],Q[:,j]), 2)\n",
    "                    for k in xrange(K):\n",
    "                        e = e + (beta/2) * ( pow(P[i][k],2) + pow(Q[k][j],2) )\n",
    "        if e < 0.001:\n",
    "            break\n",
    "    return P, Q.T"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Choosing K=2 latent features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "R = subset_hvac_normalised.values\n",
    "N = len(R)\n",
    "M = len(R[0])\n",
    "K = 2\n",
    "\n",
    "P = numpy.random.rand(N,K)\n",
    "Q = numpy.random.rand(M,K)\n",
    "\n",
    "nP, nQ = matrix_factorization(R, P, Q, K)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "pred_df = pd.DataFrame(numpy.dot(nP, nQ.T), index=subset_hvac_normalised.index, columns=subset_hvac_normalised.columns)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Reconstructed matrix on (0, 1) scale"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>hvac_5</th>\n",
       "      <th>hvac_6</th>\n",
       "      <th>hvac_7</th>\n",
       "      <th>hvac_8</th>\n",
       "      <th>hvac_9</th>\n",
       "      <th>hvac_10</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>26</th>\n",
       "      <td>0.525125</td>\n",
       "      <td>0.546110</td>\n",
       "      <td>0.564490</td>\n",
       "      <td>0.539838</td>\n",
       "      <td>0.469831</td>\n",
       "      <td>0.557974</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>59</th>\n",
       "      <td>0.195456</td>\n",
       "      <td>0.203097</td>\n",
       "      <td>0.212085</td>\n",
       "      <td>0.204079</td>\n",
       "      <td>0.176932</td>\n",
       "      <td>0.206261</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>94</th>\n",
       "      <td>0.213156</td>\n",
       "      <td>0.219759</td>\n",
       "      <td>0.251548</td>\n",
       "      <td>0.254802</td>\n",
       "      <td>0.214037</td>\n",
       "      <td>0.210379</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>101</th>\n",
       "      <td>0.271361</td>\n",
       "      <td>0.280902</td>\n",
       "      <td>0.306959</td>\n",
       "      <td>0.303246</td>\n",
       "      <td>0.258664</td>\n",
       "      <td>0.277370</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>222</th>\n",
       "      <td>0.849052</td>\n",
       "      <td>0.887698</td>\n",
       "      <td>0.857489</td>\n",
       "      <td>0.784963</td>\n",
       "      <td>0.702192</td>\n",
       "      <td>0.941851</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>364</th>\n",
       "      <td>0.517430</td>\n",
       "      <td>0.540505</td>\n",
       "      <td>0.528151</td>\n",
       "      <td>0.487253</td>\n",
       "      <td>0.433737</td>\n",
       "      <td>0.569973</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>545</th>\n",
       "      <td>0.675099</td>\n",
       "      <td>0.700375</td>\n",
       "      <td>0.745619</td>\n",
       "      <td>0.725707</td>\n",
       "      <td>0.624736</td>\n",
       "      <td>0.703016</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>624</th>\n",
       "      <td>0.134283</td>\n",
       "      <td>0.139350</td>\n",
       "      <td>0.147850</td>\n",
       "      <td>0.143617</td>\n",
       "      <td>0.123787</td>\n",
       "      <td>0.140166</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>661</th>\n",
       "      <td>0.571230</td>\n",
       "      <td>0.589567</td>\n",
       "      <td>0.666605</td>\n",
       "      <td>0.670882</td>\n",
       "      <td>0.565774</td>\n",
       "      <td>0.569187</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>871</th>\n",
       "      <td>0.204976</td>\n",
       "      <td>0.213451</td>\n",
       "      <td>0.217021</td>\n",
       "      <td>0.205434</td>\n",
       "      <td>0.179937</td>\n",
       "      <td>0.220186</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>946</th>\n",
       "      <td>0.395406</td>\n",
       "      <td>0.413291</td>\n",
       "      <td>0.400652</td>\n",
       "      <td>0.367656</td>\n",
       "      <td>0.328383</td>\n",
       "      <td>0.437676</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1169</th>\n",
       "      <td>0.136789</td>\n",
       "      <td>0.141037</td>\n",
       "      <td>0.161306</td>\n",
       "      <td>0.163323</td>\n",
       "      <td>0.137229</td>\n",
       "      <td>0.135094</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1283</th>\n",
       "      <td>0.827367</td>\n",
       "      <td>0.857147</td>\n",
       "      <td>0.927806</td>\n",
       "      <td>0.911695</td>\n",
       "      <td>0.780229</td>\n",
       "      <td>0.851507</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1310</th>\n",
       "      <td>0.261555</td>\n",
       "      <td>0.269727</td>\n",
       "      <td>0.307846</td>\n",
       "      <td>0.311355</td>\n",
       "      <td>0.261784</td>\n",
       "      <td>0.258735</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        hvac_5    hvac_6    hvac_7    hvac_8    hvac_9   hvac_10\n",
       "26    0.525125  0.546110  0.564490  0.539838  0.469831  0.557974\n",
       "59    0.195456  0.203097  0.212085  0.204079  0.176932  0.206261\n",
       "94    0.213156  0.219759  0.251548  0.254802  0.214037  0.210379\n",
       "101   0.271361  0.280902  0.306959  0.303246  0.258664  0.277370\n",
       "222   0.849052  0.887698  0.857489  0.784963  0.702192  0.941851\n",
       "364   0.517430  0.540505  0.528151  0.487253  0.433737  0.569973\n",
       "545   0.675099  0.700375  0.745619  0.725707  0.624736  0.703016\n",
       "624   0.134283  0.139350  0.147850  0.143617  0.123787  0.140166\n",
       "661   0.571230  0.589567  0.666605  0.670882  0.565774  0.569187\n",
       "871   0.204976  0.213451  0.217021  0.205434  0.179937  0.220186\n",
       "946   0.395406  0.413291  0.400652  0.367656  0.328383  0.437676\n",
       "1169  0.136789  0.141037  0.161306  0.163323  0.137229  0.135094\n",
       "1283  0.827367  0.857147  0.927806  0.911695  0.780229  0.851507\n",
       "1310  0.261555  0.269727  0.307846  0.311355  0.261784  0.258735"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pred_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "pred_26_7 = pred_df.loc[26, 'hvac_7']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "789.26356979335708"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "(col_max['hvac_7']-col_min['hvac_7'])*pred_26_7 + col_min['hvac_7']"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "Great, our method predicts energy usage to be 790 units when true consumption was 760 units. Error of 30 units only, which means the method worked really great here."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Now, I'll try to remove more entries from the matrix. Specifically, I'll remove an entry from the same month for a different home. I'll remove the entry for home #1169. Since, this entry is not the lowest one, I don't need to normalise again from scratch. The true energy for this home is ~330 units."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "subset_hvac_normalised.loc[1169, 'hvac_7'] = np.NAN"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Confirming that we have only 2 missing values in the matrix."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "assert(subset_hvac_normalised.isnull().sum().sum()==2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "numpy.random.seed(0)\n",
    "R = subset_hvac_normalised.values\n",
    "N = len(R)\n",
    "M = len(R[0])\n",
    "K = 2\n",
    "\n",
    "P = numpy.random.rand(N,K)\n",
    "Q = numpy.random.rand(M,K)\n",
    "\n",
    "nP, nQ = matrix_factorization(R, P, Q, K)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "pred_df = pd.DataFrame(numpy.dot(nP, nQ.T), index=subset_hvac_normalised.index, columns=subset_hvac_normalised.columns)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>hvac_5</th>\n",
       "      <th>hvac_6</th>\n",
       "      <th>hvac_7</th>\n",
       "      <th>hvac_8</th>\n",
       "      <th>hvac_9</th>\n",
       "      <th>hvac_10</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>26</th>\n",
       "      <td>0.525252</td>\n",
       "      <td>0.546342</td>\n",
       "      <td>0.565182</td>\n",
       "      <td>0.540056</td>\n",
       "      <td>0.470007</td>\n",
       "      <td>0.558244</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>59</th>\n",
       "      <td>0.195344</td>\n",
       "      <td>0.203010</td>\n",
       "      <td>0.212359</td>\n",
       "      <td>0.204149</td>\n",
       "      <td>0.176959</td>\n",
       "      <td>0.206131</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>94</th>\n",
       "      <td>0.213187</td>\n",
       "      <td>0.219816</td>\n",
       "      <td>0.252969</td>\n",
       "      <td>0.255115</td>\n",
       "      <td>0.214288</td>\n",
       "      <td>0.210422</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>101</th>\n",
       "      <td>0.271078</td>\n",
       "      <td>0.280661</td>\n",
       "      <td>0.307573</td>\n",
       "      <td>0.302925</td>\n",
       "      <td>0.258419</td>\n",
       "      <td>0.277217</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>222</th>\n",
       "      <td>0.849419</td>\n",
       "      <td>0.888274</td>\n",
       "      <td>0.856032</td>\n",
       "      <td>0.785051</td>\n",
       "      <td>0.702246</td>\n",
       "      <td>0.942496</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>364</th>\n",
       "      <td>0.517585</td>\n",
       "      <td>0.540781</td>\n",
       "      <td>0.527469</td>\n",
       "      <td>0.487283</td>\n",
       "      <td>0.433748</td>\n",
       "      <td>0.570288</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>545</th>\n",
       "      <td>0.674622</td>\n",
       "      <td>0.699993</td>\n",
       "      <td>0.746862</td>\n",
       "      <td>0.725564</td>\n",
       "      <td>0.624575</td>\n",
       "      <td>0.702636</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>624</th>\n",
       "      <td>0.134171</td>\n",
       "      <td>0.139255</td>\n",
       "      <td>0.148077</td>\n",
       "      <td>0.143600</td>\n",
       "      <td>0.123757</td>\n",
       "      <td>0.140059</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>661</th>\n",
       "      <td>0.570436</td>\n",
       "      <td>0.588821</td>\n",
       "      <td>0.668958</td>\n",
       "      <td>0.670553</td>\n",
       "      <td>0.565474</td>\n",
       "      <td>0.568465</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>871</th>\n",
       "      <td>0.204926</td>\n",
       "      <td>0.213444</td>\n",
       "      <td>0.216974</td>\n",
       "      <td>0.205323</td>\n",
       "      <td>0.179850</td>\n",
       "      <td>0.220218</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>946</th>\n",
       "      <td>0.395537</td>\n",
       "      <td>0.413519</td>\n",
       "      <td>0.399971</td>\n",
       "      <td>0.367628</td>\n",
       "      <td>0.328356</td>\n",
       "      <td>0.437951</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1169</th>\n",
       "      <td>0.139446</td>\n",
       "      <td>0.143789</td>\n",
       "      <td>0.165380</td>\n",
       "      <td>0.166738</td>\n",
       "      <td>0.140079</td>\n",
       "      <td>0.137696</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1283</th>\n",
       "      <td>0.826621</td>\n",
       "      <td>0.856494</td>\n",
       "      <td>0.929964</td>\n",
       "      <td>0.911630</td>\n",
       "      <td>0.780092</td>\n",
       "      <td>0.850786</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1310</th>\n",
       "      <td>0.261169</td>\n",
       "      <td>0.269388</td>\n",
       "      <td>0.308696</td>\n",
       "      <td>0.310693</td>\n",
       "      <td>0.261312</td>\n",
       "      <td>0.258608</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        hvac_5    hvac_6    hvac_7    hvac_8    hvac_9   hvac_10\n",
       "26    0.525252  0.546342  0.565182  0.540056  0.470007  0.558244\n",
       "59    0.195344  0.203010  0.212359  0.204149  0.176959  0.206131\n",
       "94    0.213187  0.219816  0.252969  0.255115  0.214288  0.210422\n",
       "101   0.271078  0.280661  0.307573  0.302925  0.258419  0.277217\n",
       "222   0.849419  0.888274  0.856032  0.785051  0.702246  0.942496\n",
       "364   0.517585  0.540781  0.527469  0.487283  0.433748  0.570288\n",
       "545   0.674622  0.699993  0.746862  0.725564  0.624575  0.702636\n",
       "624   0.134171  0.139255  0.148077  0.143600  0.123757  0.140059\n",
       "661   0.570436  0.588821  0.668958  0.670553  0.565474  0.568465\n",
       "871   0.204926  0.213444  0.216974  0.205323  0.179850  0.220218\n",
       "946   0.395537  0.413519  0.399971  0.367628  0.328356  0.437951\n",
       "1169  0.139446  0.143789  0.165380  0.166738  0.140079  0.137696\n",
       "1283  0.826621  0.856494  0.929964  0.911630  0.780092  0.850786\n",
       "1310  0.261169  0.269388  0.308696  0.310693  0.261312  0.258608"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pred_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "pred_1169_7 = pred_df.loc[1169, 'hvac_7']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "320.40357004031023"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "(col_max['hvac_7']-col_min['hvac_7'])*pred_1169_7 + col_min['hvac_7']"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We predict the usage to be 320 units when the actual consumption is about 330 units. Again, fairly good recommendation!"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Now, making the matrix much more sparse. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "subset_hvac_sparse = subset_hvac.copy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "for col in subset_hvac_sparse.columns:\n",
    "    for row in subset_hvac_sparse.index:\n",
    "        if numpy.random.random()>0.6:\n",
    "            subset_hvac_sparse.loc[row, col]=np.NaN"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "35"
      ]
     },
     "execution_count": 51,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "subset_hvac_sparse.isnull().sum().sum()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We now have 35 missing entries in the marix."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Month wise #missing entries is "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "hvac_5     7\n",
       "hvac_6     5\n",
       "hvac_7     3\n",
       "hvac_8     6\n",
       "hvac_9     8\n",
       "hvac_10    6\n",
       "dtype: int64"
      ]
     },
     "execution_count": 53,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "subset_hvac_sparse.isnull().sum()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Home wise #missing entries is"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "count    14.000000\n",
       "mean      2.500000\n",
       "std       1.506397\n",
       "min       0.000000\n",
       "25%       2.000000\n",
       "50%       3.000000\n",
       "75%       3.000000\n",
       "max       5.000000\n",
       "dtype: float64"
      ]
     },
     "execution_count": 55,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "subset_hvac_sparse.isnull().sum(axis=1).describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>hvac_5</th>\n",
       "      <th>hvac_6</th>\n",
       "      <th>hvac_7</th>\n",
       "      <th>hvac_8</th>\n",
       "      <th>hvac_9</th>\n",
       "      <th>hvac_10</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>26</th>\n",
       "      <td>242.395737</td>\n",
       "      <td>543.781433</td>\n",
       "      <td>NaN</td>\n",
       "      <td>902.418823</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>59</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>457.950378</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>94</th>\n",
       "      <td>NaN</td>\n",
       "      <td>66.621819</td>\n",
       "      <td>126.121681</td>\n",
       "      <td>NaN</td>\n",
       "      <td>92.979385</td>\n",
       "      <td>33.392750</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>101</th>\n",
       "      <td>191.965454</td>\n",
       "      <td>NaN</td>\n",
       "      <td>462.708923</td>\n",
       "      <td>NaN</td>\n",
       "      <td>350.491455</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>222</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>887.044739</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>364</th>\n",
       "      <td>236.925064</td>\n",
       "      <td>535.010376</td>\n",
       "      <td>736.468933</td>\n",
       "      <td>850.080811</td>\n",
       "      <td>642.858459</td>\n",
       "      <td>378.799713</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>545</th>\n",
       "      <td>350.414551</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1065.381104</td>\n",
       "      <td>1120.907593</td>\n",
       "      <td>NaN</td>\n",
       "      <td>375.303040</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>624</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>307.978149</td>\n",
       "      <td>389.598602</td>\n",
       "      <td>NaN</td>\n",
       "      <td>22.099518</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>661</th>\n",
       "      <td>221.594315</td>\n",
       "      <td>648.794067</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>756.477905</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>871</th>\n",
       "      <td>84.128731</td>\n",
       "      <td>231.698425</td>\n",
       "      <td>403.408813</td>\n",
       "      <td>514.838928</td>\n",
       "      <td>NaN</td>\n",
       "      <td>140.866745</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>946</th>\n",
       "      <td>217.752716</td>\n",
       "      <td>446.865326</td>\n",
       "      <td>605.236755</td>\n",
       "      <td>683.144165</td>\n",
       "      <td>452.278839</td>\n",
       "      <td>248.735733</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1169</th>\n",
       "      <td>NaN</td>\n",
       "      <td>177.999435</td>\n",
       "      <td>327.977417</td>\n",
       "      <td>445.256683</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1283</th>\n",
       "      <td>NaN</td>\n",
       "      <td>526.241333</td>\n",
       "      <td>1300.884033</td>\n",
       "      <td>1507.603027</td>\n",
       "      <td>NaN</td>\n",
       "      <td>452.018646</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1310</th>\n",
       "      <td>NaN</td>\n",
       "      <td>300.484253</td>\n",
       "      <td>338.782837</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>273.744385</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "          hvac_5      hvac_6       hvac_7       hvac_8      hvac_9     hvac_10\n",
       "26    242.395737  543.781433          NaN   902.418823         NaN         NaN\n",
       "59           NaN         NaN   457.950378          NaN         NaN         NaN\n",
       "94           NaN   66.621819   126.121681          NaN   92.979385   33.392750\n",
       "101   191.965454         NaN   462.708923          NaN  350.491455         NaN\n",
       "222          NaN         NaN          NaN          NaN  887.044739         NaN\n",
       "364   236.925064  535.010376   736.468933   850.080811  642.858459  378.799713\n",
       "545   350.414551         NaN  1065.381104  1120.907593         NaN  375.303040\n",
       "624          NaN         NaN   307.978149   389.598602         NaN   22.099518\n",
       "661   221.594315  648.794067          NaN          NaN  756.477905         NaN\n",
       "871    84.128731  231.698425   403.408813   514.838928         NaN  140.866745\n",
       "946   217.752716  446.865326   605.236755   683.144165  452.278839  248.735733\n",
       "1169         NaN  177.999435   327.977417   445.256683         NaN         NaN\n",
       "1283         NaN  526.241333  1300.884033  1507.603027         NaN  452.018646\n",
       "1310         NaN  300.484253   338.782837          NaN         NaN  273.744385"
      ]
     },
     "execution_count": 56,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "subset_hvac_sparse"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Now, normalising this matrix."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "col_max = subset_hvac_sparse.max()\n",
    "col_min = subset_hvac_sparse.min()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "subset_hvac_sparse_normalised = subset_hvac_sparse.copy()\n",
    "for col in subset_hvac.columns:\n",
    "    subset_hvac_sparse_normalised[col] = (subset_hvac_sparse[col]-col_min[col])/(col_max[col]-col_min[col])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>hvac_5</th>\n",
       "      <th>hvac_6</th>\n",
       "      <th>hvac_7</th>\n",
       "      <th>hvac_8</th>\n",
       "      <th>hvac_9</th>\n",
       "      <th>hvac_10</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>26</th>\n",
       "      <td>0.594350</td>\n",
       "      <td>0.819619</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.458692</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>59</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.282465</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>94</th>\n",
       "      <td>NaN</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.026268</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>101</th>\n",
       "      <td>0.404966</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.286515</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.324296</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>222</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>364</th>\n",
       "      <td>0.573806</td>\n",
       "      <td>0.804553</td>\n",
       "      <td>0.519550</td>\n",
       "      <td>0.411879</td>\n",
       "      <td>0.692486</td>\n",
       "      <td>0.829691</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>545</th>\n",
       "      <td>1.000000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.799531</td>\n",
       "      <td>0.654120</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.821558</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>624</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.154803</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>661</th>\n",
       "      <td>0.516233</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.835572</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>871</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.283553</td>\n",
       "      <td>0.236037</td>\n",
       "      <td>0.112021</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.276255</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>946</th>\n",
       "      <td>0.501807</td>\n",
       "      <td>0.653146</td>\n",
       "      <td>0.407840</td>\n",
       "      <td>0.262562</td>\n",
       "      <td>0.452481</td>\n",
       "      <td>0.527160</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1169</th>\n",
       "      <td>NaN</td>\n",
       "      <td>0.191314</td>\n",
       "      <td>0.171827</td>\n",
       "      <td>0.049783</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1283</th>\n",
       "      <td>NaN</td>\n",
       "      <td>0.789491</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1310</th>\n",
       "      <td>NaN</td>\n",
       "      <td>0.401707</td>\n",
       "      <td>0.181025</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.585331</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        hvac_5    hvac_6    hvac_7    hvac_8    hvac_9   hvac_10\n",
       "26    0.594350  0.819619       NaN  0.458692       NaN       NaN\n",
       "59         NaN       NaN  0.282465       NaN       NaN       NaN\n",
       "94         NaN  0.000000  0.000000       NaN  0.000000  0.026268\n",
       "101   0.404966       NaN  0.286515       NaN  0.324296       NaN\n",
       "222        NaN       NaN       NaN       NaN  1.000000       NaN\n",
       "364   0.573806  0.804553  0.519550  0.411879  0.692486  0.829691\n",
       "545   1.000000       NaN  0.799531  0.654120       NaN  0.821558\n",
       "624        NaN       NaN  0.154803  0.000000       NaN  0.000000\n",
       "661   0.516233  1.000000       NaN       NaN  0.835572       NaN\n",
       "871   0.000000  0.283553  0.236037  0.112021       NaN  0.276255\n",
       "946   0.501807  0.653146  0.407840  0.262562  0.452481  0.527160\n",
       "1169       NaN  0.191314  0.171827  0.049783       NaN       NaN\n",
       "1283       NaN  0.789491  1.000000  1.000000       NaN  1.000000\n",
       "1310       NaN  0.401707  0.181025       NaN       NaN  0.585331"
      ]
     },
     "execution_count": 59,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "subset_hvac_sparse_normalised"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "numpy.random.seed(0)\n",
    "R = subset_hvac_sparse_normalised.values\n",
    "N = len(R)\n",
    "M = len(R[0])\n",
    "K = 2\n",
    "\n",
    "P = numpy.random.rand(N,K)\n",
    "Q = numpy.random.rand(M,K)\n",
    "\n",
    "nP, nQ = matrix_factorization(R, P, Q, K)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "pred_df = pd.DataFrame(numpy.dot(nP, nQ.T), index=subset_hvac_normalised.index, columns=subset_hvac_normalised.columns)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>hvac_5</th>\n",
       "      <th>hvac_6</th>\n",
       "      <th>hvac_7</th>\n",
       "      <th>hvac_8</th>\n",
       "      <th>hvac_9</th>\n",
       "      <th>hvac_10</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>26</th>\n",
       "      <td>0.626751</td>\n",
       "      <td>0.725728</td>\n",
       "      <td>0.574429</td>\n",
       "      <td>0.507956</td>\n",
       "      <td>0.723713</td>\n",
       "      <td>0.715155</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>59</th>\n",
       "      <td>0.446145</td>\n",
       "      <td>0.471976</td>\n",
       "      <td>0.372976</td>\n",
       "      <td>0.255860</td>\n",
       "      <td>0.509112</td>\n",
       "      <td>0.500437</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>94</th>\n",
       "      <td>0.166016</td>\n",
       "      <td>0.232958</td>\n",
       "      <td>0.184941</td>\n",
       "      <td>0.231032</td>\n",
       "      <td>0.197225</td>\n",
       "      <td>0.197316</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>101</th>\n",
       "      <td>0.331219</td>\n",
       "      <td>0.432536</td>\n",
       "      <td>0.343024</td>\n",
       "      <td>0.384554</td>\n",
       "      <td>0.389111</td>\n",
       "      <td>0.387425</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>222</th>\n",
       "      <td>0.801248</td>\n",
       "      <td>0.779461</td>\n",
       "      <td>0.614957</td>\n",
       "      <td>0.297987</td>\n",
       "      <td>0.905082</td>\n",
       "      <td>0.885557</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>364</th>\n",
       "      <td>0.650821</td>\n",
       "      <td>0.702224</td>\n",
       "      <td>0.555131</td>\n",
       "      <td>0.405750</td>\n",
       "      <td>0.744536</td>\n",
       "      <td>0.732676</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>545</th>\n",
       "      <td>0.817796</td>\n",
       "      <td>0.958538</td>\n",
       "      <td>0.758860</td>\n",
       "      <td>0.690261</td>\n",
       "      <td>0.945887</td>\n",
       "      <td>0.935392</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>624</th>\n",
       "      <td>0.121631</td>\n",
       "      <td>0.140518</td>\n",
       "      <td>0.111219</td>\n",
       "      <td>0.097817</td>\n",
       "      <td>0.140404</td>\n",
       "      <td>0.138725</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>661</th>\n",
       "      <td>0.642869</td>\n",
       "      <td>0.870370</td>\n",
       "      <td>0.690618</td>\n",
       "      <td>0.819481</td>\n",
       "      <td>0.759418</td>\n",
       "      <td>0.757931</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>871</th>\n",
       "      <td>0.283462</td>\n",
       "      <td>0.285528</td>\n",
       "      <td>0.225425</td>\n",
       "      <td>0.128576</td>\n",
       "      <td>0.321522</td>\n",
       "      <td>0.315180</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>946</th>\n",
       "      <td>0.482190</td>\n",
       "      <td>0.513039</td>\n",
       "      <td>0.405469</td>\n",
       "      <td>0.283476</td>\n",
       "      <td>0.550641</td>\n",
       "      <td>0.541436</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1169</th>\n",
       "      <td>0.228538</td>\n",
       "      <td>0.228856</td>\n",
       "      <td>0.180661</td>\n",
       "      <td>0.100469</td>\n",
       "      <td>0.259041</td>\n",
       "      <td>0.253850</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1283</th>\n",
       "      <td>0.800990</td>\n",
       "      <td>1.021473</td>\n",
       "      <td>0.809787</td>\n",
       "      <td>0.871845</td>\n",
       "      <td>0.937661</td>\n",
       "      <td>0.932164</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1310</th>\n",
       "      <td>0.337451</td>\n",
       "      <td>0.462984</td>\n",
       "      <td>0.367438</td>\n",
       "      <td>0.444643</td>\n",
       "      <td>0.399459</td>\n",
       "      <td>0.399033</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        hvac_5    hvac_6    hvac_7    hvac_8    hvac_9   hvac_10\n",
       "26    0.626751  0.725728  0.574429  0.507956  0.723713  0.715155\n",
       "59    0.446145  0.471976  0.372976  0.255860  0.509112  0.500437\n",
       "94    0.166016  0.232958  0.184941  0.231032  0.197225  0.197316\n",
       "101   0.331219  0.432536  0.343024  0.384554  0.389111  0.387425\n",
       "222   0.801248  0.779461  0.614957  0.297987  0.905082  0.885557\n",
       "364   0.650821  0.702224  0.555131  0.405750  0.744536  0.732676\n",
       "545   0.817796  0.958538  0.758860  0.690261  0.945887  0.935392\n",
       "624   0.121631  0.140518  0.111219  0.097817  0.140404  0.138725\n",
       "661   0.642869  0.870370  0.690618  0.819481  0.759418  0.757931\n",
       "871   0.283462  0.285528  0.225425  0.128576  0.321522  0.315180\n",
       "946   0.482190  0.513039  0.405469  0.283476  0.550641  0.541436\n",
       "1169  0.228538  0.228856  0.180661  0.100469  0.259041  0.253850\n",
       "1283  0.800990  1.021473  0.809787  0.871845  0.937661  0.932164\n",
       "1310  0.337451  0.462984  0.367438  0.444643  0.399459  0.399033"
      ]
     },
     "execution_count": 62,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pred_df"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Now, let us pick up a few entries. Home #59 had 4/5 missing entries!"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "home_pred = {}\n",
    "home = 59\n",
    "home_pred[home] = {}\n",
    "for month in range(5, 11):\n",
    "    home_pred[59][month] = (col_max['hvac_%d' %month]-col_min['hvac_%d' %month])*pred_df.loc[home,'hvac_%d'%month] +col_min['hvac_%d' %month]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "pred_59 = pd.DataFrame(home_pred).squeeze()\n",
    "gt_59 = subset_hvac.ix[59]\n",
    "gt_59.index=pred_59.index"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Here's the GT and prediction for this home. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>gt</th>\n",
       "      <th>pred</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>77.613205</td>\n",
       "      <td>202.930850</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>197.951508</td>\n",
       "      <td>341.393123</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>457.950378</td>\n",
       "      <td>564.280025</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>700.443054</td>\n",
       "      <td>675.651405</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>244.340973</td>\n",
       "      <td>497.247409</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>81.064682</td>\n",
       "      <td>237.247047</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "            gt        pred\n",
       "5    77.613205  202.930850\n",
       "6   197.951508  341.393123\n",
       "7   457.950378  564.280025\n",
       "8   700.443054  675.651405\n",
       "9   244.340973  497.247409\n",
       "10   81.064682  237.247047"
      ]
     },
     "execution_count": 76,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pd.DataFrame({\"gt\":gt_59, \"pred\":pred_59})"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Given that we had 1/6 records for home #59. Our predictions aren't all that bad."
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.11"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}