notebooks/BostonSplit.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h2> Import Libraries</h2>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "%matplotlib inline\n",
    "\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "from sklearn.datasets import load_boston\n",
    "\n",
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "from sklearn.linear_model import LinearRegression"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "## Load the Data\n",
    "The boston house-price dataset is one of datasets scikit-learn comes with that do not require the downloading of any file from some external website. The code below loads the boston dataset."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>CRIM</th>\n",
       "      <th>ZN</th>\n",
       "      <th>INDUS</th>\n",
       "      <th>CHAS</th>\n",
       "      <th>NOX</th>\n",
       "      <th>RM</th>\n",
       "      <th>AGE</th>\n",
       "      <th>DIS</th>\n",
       "      <th>RAD</th>\n",
       "      <th>TAX</th>\n",
       "      <th>PTRATIO</th>\n",
       "      <th>B</th>\n",
       "      <th>LSTAT</th>\n",
       "      <th>target</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>0.00632</td>\n",
       "      <td>18.0</td>\n",
       "      <td>2.31</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.538</td>\n",
       "      <td>6.575</td>\n",
       "      <td>65.2</td>\n",
       "      <td>4.0900</td>\n",
       "      <td>1.0</td>\n",
       "      <td>296.0</td>\n",
       "      <td>15.3</td>\n",
       "      <td>396.90</td>\n",
       "      <td>4.98</td>\n",
       "      <td>24.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>0.02731</td>\n",
       "      <td>0.0</td>\n",
       "      <td>7.07</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.469</td>\n",
       "      <td>6.421</td>\n",
       "      <td>78.9</td>\n",
       "      <td>4.9671</td>\n",
       "      <td>2.0</td>\n",
       "      <td>242.0</td>\n",
       "      <td>17.8</td>\n",
       "      <td>396.90</td>\n",
       "      <td>9.14</td>\n",
       "      <td>21.6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>0.02729</td>\n",
       "      <td>0.0</td>\n",
       "      <td>7.07</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.469</td>\n",
       "      <td>7.185</td>\n",
       "      <td>61.1</td>\n",
       "      <td>4.9671</td>\n",
       "      <td>2.0</td>\n",
       "      <td>242.0</td>\n",
       "      <td>17.8</td>\n",
       "      <td>392.83</td>\n",
       "      <td>4.03</td>\n",
       "      <td>34.7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>0.03237</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.18</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.458</td>\n",
       "      <td>6.998</td>\n",
       "      <td>45.8</td>\n",
       "      <td>6.0622</td>\n",
       "      <td>3.0</td>\n",
       "      <td>222.0</td>\n",
       "      <td>18.7</td>\n",
       "      <td>394.63</td>\n",
       "      <td>2.94</td>\n",
       "      <td>33.4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>0.06905</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.18</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.458</td>\n",
       "      <td>7.147</td>\n",
       "      <td>54.2</td>\n",
       "      <td>6.0622</td>\n",
       "      <td>3.0</td>\n",
       "      <td>222.0</td>\n",
       "      <td>18.7</td>\n",
       "      <td>396.90</td>\n",
       "      <td>5.33</td>\n",
       "      <td>36.2</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      CRIM    ZN  INDUS  CHAS    NOX     RM   AGE     DIS  RAD    TAX  \\\n",
       "0  0.00632  18.0   2.31   0.0  0.538  6.575  65.2  4.0900  1.0  296.0   \n",
       "1  0.02731   0.0   7.07   0.0  0.469  6.421  78.9  4.9671  2.0  242.0   \n",
       "2  0.02729   0.0   7.07   0.0  0.469  7.185  61.1  4.9671  2.0  242.0   \n",
       "3  0.03237   0.0   2.18   0.0  0.458  6.998  45.8  6.0622  3.0  222.0   \n",
       "4  0.06905   0.0   2.18   0.0  0.458  7.147  54.2  6.0622  3.0  222.0   \n",
       "\n",
       "   PTRATIO       B  LSTAT  target  \n",
       "0     15.3  396.90   4.98    24.0  \n",
       "1     17.8  396.90   9.14    21.6  \n",
       "2     17.8  392.83   4.03    34.7  \n",
       "3     18.7  394.63   2.94    33.4  \n",
       "4     18.7  396.90   5.33    36.2  "
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data = load_boston()\n",
    "df = pd.DataFrame(data.data, columns=data.feature_names)\n",
    "df['target'] = data.target\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h2> Remove Missing or Impute Values</h2>\n",
    "If you want to build models with your data, null values are (almost) never allowed. It is important to always see how many samples have missing values and for which columns."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(506, 14)"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Look at the shape of the dataframe\n",
    "df.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "CRIM       0\n",
       "ZN         0\n",
       "INDUS      0\n",
       "CHAS       0\n",
       "NOX        0\n",
       "RM         0\n",
       "AGE        0\n",
       "DIS        0\n",
       "RAD        0\n",
       "TAX        0\n",
       "PTRATIO    0\n",
       "B          0\n",
       "LSTAT      0\n",
       "target     0\n",
       "dtype: int64"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# There are no missing values in the dataset\n",
    "df.isnull().sum()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h2> Arrange Data into Features Matrix and Target Vector </h2>\n",
    "What we are predicing is the continuous column \"target\" which is the median value of owner-occupied homes in $1000’s. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "X = df.loc[:, ['RM', 'LSTAT', 'PTRATIO']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "y = df.loc[:, 'target']"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Splitting Data into Training and Test Sets\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Original random state is 2\n",
    "X_train, X_test, y_train, y_test = train_test_split(X, y, random_state=2)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Train Test Split Visualization\n",
    "\n",
    "A relatively new feature of pandas is conditional formatting. https://pandas.pydata.org/pandas-docs/stable/user_guide/style.html"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train = pd.DataFrame(X_train, columns=['RM', 'LSTAT', 'PTRATIO'])\n",
    "\n",
    "X_test = pd.DataFrame(X_test, columns=['RM', 'LSTAT', 'PTRATIO'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train['split'] = 'train'\n",
    "X_test['split'] = 'test'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>RM</th>\n",
       "      <th>LSTAT</th>\n",
       "      <th>PTRATIO</th>\n",
       "      <th>split</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>312</td>\n",
       "      <td>6.023</td>\n",
       "      <td>11.72</td>\n",
       "      <td>18.4</td>\n",
       "      <td>train</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>328</td>\n",
       "      <td>5.868</td>\n",
       "      <td>9.97</td>\n",
       "      <td>16.9</td>\n",
       "      <td>train</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>251</td>\n",
       "      <td>6.438</td>\n",
       "      <td>3.59</td>\n",
       "      <td>19.1</td>\n",
       "      <td>train</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>205</td>\n",
       "      <td>5.891</td>\n",
       "      <td>10.87</td>\n",
       "      <td>18.6</td>\n",
       "      <td>train</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>231</td>\n",
       "      <td>7.412</td>\n",
       "      <td>5.25</td>\n",
       "      <td>17.4</td>\n",
       "      <td>train</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>22</td>\n",
       "      <td>6.142</td>\n",
       "      <td>18.72</td>\n",
       "      <td>21.0</td>\n",
       "      <td>train</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>72</td>\n",
       "      <td>6.065</td>\n",
       "      <td>5.52</td>\n",
       "      <td>19.2</td>\n",
       "      <td>train</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>493</td>\n",
       "      <td>5.707</td>\n",
       "      <td>12.01</td>\n",
       "      <td>19.2</td>\n",
       "      <td>train</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>15</td>\n",
       "      <td>5.834</td>\n",
       "      <td>8.47</td>\n",
       "      <td>21.0</td>\n",
       "      <td>train</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>168</td>\n",
       "      <td>6.319</td>\n",
       "      <td>11.10</td>\n",
       "      <td>14.7</td>\n",
       "      <td>train</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>379 rows × 4 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "        RM  LSTAT  PTRATIO  split\n",
       "312  6.023  11.72     18.4  train\n",
       "328  5.868   9.97     16.9  train\n",
       "251  6.438   3.59     19.1  train\n",
       "205  5.891  10.87     18.6  train\n",
       "231  7.412   5.25     17.4  train\n",
       "..     ...    ...      ...    ...\n",
       "22   6.142  18.72     21.0  train\n",
       "72   6.065   5.52     19.2  train\n",
       "493  5.707  12.01     19.2  train\n",
       "15   5.834   8.47     21.0  train\n",
       "168  6.319  11.10     14.7  train\n",
       "\n",
       "[379 rows x 4 columns]"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_train"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train['target'] = y_train\n",
    "X_test['target'] = y_test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "fullDF = pd.concat([X_train, X_test], axis = 0, ignore_index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>RM</th>\n",
       "      <th>LSTAT</th>\n",
       "      <th>PTRATIO</th>\n",
       "      <th>split</th>\n",
       "      <th>target</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>312</td>\n",
       "      <td>6.023</td>\n",
       "      <td>11.72</td>\n",
       "      <td>18.4</td>\n",
       "      <td>train</td>\n",
       "      <td>19.4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>328</td>\n",
       "      <td>5.868</td>\n",
       "      <td>9.97</td>\n",
       "      <td>16.9</td>\n",
       "      <td>train</td>\n",
       "      <td>19.3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>251</td>\n",
       "      <td>6.438</td>\n",
       "      <td>3.59</td>\n",
       "      <td>19.1</td>\n",
       "      <td>train</td>\n",
       "      <td>24.8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>205</td>\n",
       "      <td>5.891</td>\n",
       "      <td>10.87</td>\n",
       "      <td>18.6</td>\n",
       "      <td>train</td>\n",
       "      <td>22.6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>231</td>\n",
       "      <td>7.412</td>\n",
       "      <td>5.25</td>\n",
       "      <td>17.4</td>\n",
       "      <td>train</td>\n",
       "      <td>31.7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>114</td>\n",
       "      <td>6.254</td>\n",
       "      <td>10.45</td>\n",
       "      <td>17.8</td>\n",
       "      <td>train</td>\n",
       "      <td>18.5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>437</td>\n",
       "      <td>6.152</td>\n",
       "      <td>26.45</td>\n",
       "      <td>20.2</td>\n",
       "      <td>train</td>\n",
       "      <td>8.7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>68</td>\n",
       "      <td>5.594</td>\n",
       "      <td>13.09</td>\n",
       "      <td>18.9</td>\n",
       "      <td>train</td>\n",
       "      <td>17.4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>214</td>\n",
       "      <td>5.412</td>\n",
       "      <td>29.55</td>\n",
       "      <td>18.6</td>\n",
       "      <td>train</td>\n",
       "      <td>23.7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>377</td>\n",
       "      <td>6.794</td>\n",
       "      <td>21.24</td>\n",
       "      <td>20.2</td>\n",
       "      <td>train</td>\n",
       "      <td>13.3</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        RM  LSTAT  PTRATIO  split  target\n",
       "312  6.023  11.72     18.4  train    19.4\n",
       "328  5.868   9.97     16.9  train    19.3\n",
       "251  6.438   3.59     19.1  train    24.8\n",
       "205  5.891  10.87     18.6  train    22.6\n",
       "231  7.412   5.25     17.4  train    31.7\n",
       "114  6.254  10.45     17.8  train    18.5\n",
       "437  6.152  26.45     20.2  train     8.7\n",
       "68   5.594  13.09     18.9  train    17.4\n",
       "214  5.412  29.55     18.6  train    23.7\n",
       "377  6.794  21.24     20.2  train    13.3"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "fullDF.head(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "506"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(fullDF.index)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "506"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(np.unique(fullDF.index))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "fullDFsplit = fullDF.copy()\n",
    "fullDF = fullDF.drop(columns = ['split'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<style  type=\"text/css\" >\n",
       "    #T_afd8a6c0_0fff_11eb_a306_b8f6b11350ebrow0_col0 {\n",
       "            background-color:  #40E0D0;\n",
       "            border-color:  black;\n",
       "            border:  1px solid black;\n",
       "        }    #T_afd8a6c0_0fff_11eb_a306_b8f6b11350ebrow0_col1 {\n",
       "            background-color:  #40E0D0;\n",
       "            border-color:  black;\n",
       "            border:  1px solid black;\n",
       "        }    #T_afd8a6c0_0fff_11eb_a306_b8f6b11350ebrow0_col2 {\n",
       "            background-color:  #40E0D0;\n",
       "            border-color:  black;\n",
       "            border:  1px solid black;\n",
       "        }    #T_afd8a6c0_0fff_11eb_a306_b8f6b11350ebrow0_col3 {\n",
       "            background-color:  #FFD700;\n",
       "            border-color:  black;\n",
       "            border:  1px solid black;\n",
       "        }    #T_afd8a6c0_0fff_11eb_a306_b8f6b11350ebrow1_col0 {\n",
       "            background-color:  #00FFFF;\n",
       "            border-color:  black;\n",
       "            border:  1px solid black;\n",
       "        }    #T_afd8a6c0_0fff_11eb_a306_b8f6b11350ebrow1_col1 {\n",
       "            background-color:  #00FFFF;\n",
       "            border-color:  black;\n",
       "            border:  1px solid black;\n",
       "        }    #T_afd8a6c0_0fff_11eb_a306_b8f6b11350ebrow1_col2 {\n",
       "            background-color:  #00FFFF;\n",
       "            border-color:  black;\n",
       "            border:  1px solid black;\n",
       "        }    #T_afd8a6c0_0fff_11eb_a306_b8f6b11350ebrow1_col3 {\n",
       "            background-color:  #FFFF00;\n",
       "            border-color:  black;\n",
       "            border:  1px solid black;\n",
       "        }    #T_afd8a6c0_0fff_11eb_a306_b8f6b11350ebrow2_col0 {\n",
       "            background-color:  #00FFFF;\n",
       "            border-color:  black;\n",
       "            border:  1px solid black;\n",
       "        }    #T_afd8a6c0_0fff_11eb_a306_b8f6b11350ebrow2_col1 {\n",
       "            background-color:  #00FFFF;\n",
       "            border-color:  black;\n",
       "            border:  1px solid black;\n",
       "        }    #T_afd8a6c0_0fff_11eb_a306_b8f6b11350ebrow2_col2 {\n",
       "            background-color:  #00FFFF;\n",
       "            border-color:  black;\n",
       "            border:  1px solid black;\n",
       "        }    #T_afd8a6c0_0fff_11eb_a306_b8f6b11350ebrow2_col3 {\n",
       "            background-color:  #FFFF00;\n",
       "            border-color:  black;\n",
       "            border:  1px solid black;\n",
       "        }    #T_afd8a6c0_0fff_11eb_a306_b8f6b11350ebrow3_col0 {\n",
       "            background-color:  #40E0D0;\n",
       "            border-color:  black;\n",
       "            border:  1px solid black;\n",
       "        }    #T_afd8a6c0_0fff_11eb_a306_b8f6b11350ebrow3_col1 {\n",
       "            background-color:  #40E0D0;\n",
       "            border-color:  black;\n",
       "            border:  1px solid black;\n",
       "        }    #T_afd8a6c0_0fff_11eb_a306_b8f6b11350ebrow3_col2 {\n",
       "            background-color:  #40E0D0;\n",
       "            border-color:  black;\n",
       "            border:  1px solid black;\n",
       "        }    #T_afd8a6c0_0fff_11eb_a306_b8f6b11350ebrow3_col3 {\n",
       "            background-color:  #FFD700;\n",
       "            border-color:  black;\n",
       "            border:  1px solid black;\n",
       "        }    #T_afd8a6c0_0fff_11eb_a306_b8f6b11350ebrow4_col0 {\n",
       "            background-color:  #40E0D0;\n",
       "            border-color:  black;\n",
       "            border:  1px solid black;\n",
       "        }    #T_afd8a6c0_0fff_11eb_a306_b8f6b11350ebrow4_col1 {\n",
       "            background-color:  #40E0D0;\n",
       "            border-color:  black;\n",
       "            border:  1px solid black;\n",
       "        }    #T_afd8a6c0_0fff_11eb_a306_b8f6b11350ebrow4_col2 {\n",
       "            background-color:  #40E0D0;\n",
       "            border-color:  black;\n",
       "            border:  1px solid black;\n",
       "        }    #T_afd8a6c0_0fff_11eb_a306_b8f6b11350ebrow4_col3 {\n",
       "            background-color:  #FFD700;\n",
       "            border-color:  black;\n",
       "            border:  1px solid black;\n",
       "        }    #T_afd8a6c0_0fff_11eb_a306_b8f6b11350ebrow5_col0 {\n",
       "            background-color:  #40E0D0;\n",
       "            border-color:  black;\n",
       "            border:  1px solid black;\n",
       "        }    #T_afd8a6c0_0fff_11eb_a306_b8f6b11350ebrow5_col1 {\n",
       "            background-color:  #40E0D0;\n",
       "            border-color:  black;\n",
       "            border:  1px solid black;\n",
       "        }    #T_afd8a6c0_0fff_11eb_a306_b8f6b11350ebrow5_col2 {\n",
       "            background-color:  #40E0D0;\n",
       "            border-color:  black;\n",
       "            border:  1px solid black;\n",
       "        }    #T_afd8a6c0_0fff_11eb_a306_b8f6b11350ebrow5_col3 {\n",
       "            background-color:  #FFD700;\n",
       "            border-color:  black;\n",
       "            border:  1px solid black;\n",
       "        }    #T_afd8a6c0_0fff_11eb_a306_b8f6b11350ebrow6_col0 {\n",
       "            background-color:  #40E0D0;\n",
       "            border-color:  black;\n",
       "            border:  1px solid black;\n",
       "        }    #T_afd8a6c0_0fff_11eb_a306_b8f6b11350ebrow6_col1 {\n",
       "            background-color:  #40E0D0;\n",
       "            border-color:  black;\n",
       "            border:  1px solid black;\n",
       "        }    #T_afd8a6c0_0fff_11eb_a306_b8f6b11350ebrow6_col2 {\n",
       "            background-color:  #40E0D0;\n",
       "            border-color:  black;\n",
       "            border:  1px solid black;\n",
       "        }    #T_afd8a6c0_0fff_11eb_a306_b8f6b11350ebrow6_col3 {\n",
       "            background-color:  #FFD700;\n",
       "            border-color:  black;\n",
       "            border:  1px solid black;\n",
       "        }    #T_afd8a6c0_0fff_11eb_a306_b8f6b11350ebrow7_col0 {\n",
       "            background-color:  #00FFFF;\n",
       "            border-color:  black;\n",
       "            border:  1px solid black;\n",
       "        }    #T_afd8a6c0_0fff_11eb_a306_b8f6b11350ebrow7_col1 {\n",
       "            background-color:  #00FFFF;\n",
       "            border-color:  black;\n",
       "            border:  1px solid black;\n",
       "        }    #T_afd8a6c0_0fff_11eb_a306_b8f6b11350ebrow7_col2 {\n",
       "            background-color:  #00FFFF;\n",
       "            border-color:  black;\n",
       "            border:  1px solid black;\n",
       "        }    #T_afd8a6c0_0fff_11eb_a306_b8f6b11350ebrow7_col3 {\n",
       "            background-color:  #FFFF00;\n",
       "            border-color:  black;\n",
       "            border:  1px solid black;\n",
       "        }    #T_afd8a6c0_0fff_11eb_a306_b8f6b11350ebrow8_col0 {\n",
       "            background-color:  #40E0D0;\n",
       "            border-color:  black;\n",
       "            border:  1px solid black;\n",
       "        }    #T_afd8a6c0_0fff_11eb_a306_b8f6b11350ebrow8_col1 {\n",
       "            background-color:  #40E0D0;\n",
       "            border-color:  black;\n",
       "            border:  1px solid black;\n",
       "        }    #T_afd8a6c0_0fff_11eb_a306_b8f6b11350ebrow8_col2 {\n",
       "            background-color:  #40E0D0;\n",
       "            border-color:  black;\n",
       "            border:  1px solid black;\n",
       "        }    #T_afd8a6c0_0fff_11eb_a306_b8f6b11350ebrow8_col3 {\n",
       "            background-color:  #FFD700;\n",
       "            border-color:  black;\n",
       "            border:  1px solid black;\n",
       "        }    #T_afd8a6c0_0fff_11eb_a306_b8f6b11350ebrow9_col0 {\n",
       "            background-color:  #40E0D0;\n",
       "            border-color:  black;\n",
       "            border:  1px solid black;\n",
       "        }    #T_afd8a6c0_0fff_11eb_a306_b8f6b11350ebrow9_col1 {\n",
       "            background-color:  #40E0D0;\n",
       "            border-color:  black;\n",
       "            border:  1px solid black;\n",
       "        }    #T_afd8a6c0_0fff_11eb_a306_b8f6b11350ebrow9_col2 {\n",
       "            background-color:  #40E0D0;\n",
       "            border-color:  black;\n",
       "            border:  1px solid black;\n",
       "        }    #T_afd8a6c0_0fff_11eb_a306_b8f6b11350ebrow9_col3 {\n",
       "            background-color:  #FFD700;\n",
       "            border-color:  black;\n",
       "            border:  1px solid black;\n",
       "        }</style><table id=\"T_afd8a6c0_0fff_11eb_a306_b8f6b11350eb\" ><thead>    <tr>        <th class=\"blank level0\" ></th>        <th class=\"col_heading level0 col0\" >RM</th>        <th class=\"col_heading level0 col1\" >LSTAT</th>        <th class=\"col_heading level0 col2\" >PTRATIO</th>        <th class=\"col_heading level0 col3\" >target</th>    </tr></thead><tbody>\n",
       "                <tr>\n",
       "                        <th id=\"T_afd8a6c0_0fff_11eb_a306_b8f6b11350eblevel0_row0\" class=\"row_heading level0 row0\" >0</th>\n",
       "                        <td id=\"T_afd8a6c0_0fff_11eb_a306_b8f6b11350ebrow0_col0\" class=\"data row0 col0\" >6.575</td>\n",
       "                        <td id=\"T_afd8a6c0_0fff_11eb_a306_b8f6b11350ebrow0_col1\" class=\"data row0 col1\" >4.98</td>\n",
       "                        <td id=\"T_afd8a6c0_0fff_11eb_a306_b8f6b11350ebrow0_col2\" class=\"data row0 col2\" >15.3</td>\n",
       "                        <td id=\"T_afd8a6c0_0fff_11eb_a306_b8f6b11350ebrow0_col3\" class=\"data row0 col3\" >24</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_afd8a6c0_0fff_11eb_a306_b8f6b11350eblevel0_row1\" class=\"row_heading level0 row1\" >1</th>\n",
       "                        <td id=\"T_afd8a6c0_0fff_11eb_a306_b8f6b11350ebrow1_col0\" class=\"data row1 col0\" >6.421</td>\n",
       "                        <td id=\"T_afd8a6c0_0fff_11eb_a306_b8f6b11350ebrow1_col1\" class=\"data row1 col1\" >9.14</td>\n",
       "                        <td id=\"T_afd8a6c0_0fff_11eb_a306_b8f6b11350ebrow1_col2\" class=\"data row1 col2\" >17.8</td>\n",
       "                        <td id=\"T_afd8a6c0_0fff_11eb_a306_b8f6b11350ebrow1_col3\" class=\"data row1 col3\" >21.6</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_afd8a6c0_0fff_11eb_a306_b8f6b11350eblevel0_row2\" class=\"row_heading level0 row2\" >2</th>\n",
       "                        <td id=\"T_afd8a6c0_0fff_11eb_a306_b8f6b11350ebrow2_col0\" class=\"data row2 col0\" >7.185</td>\n",
       "                        <td id=\"T_afd8a6c0_0fff_11eb_a306_b8f6b11350ebrow2_col1\" class=\"data row2 col1\" >4.03</td>\n",
       "                        <td id=\"T_afd8a6c0_0fff_11eb_a306_b8f6b11350ebrow2_col2\" class=\"data row2 col2\" >17.8</td>\n",
       "                        <td id=\"T_afd8a6c0_0fff_11eb_a306_b8f6b11350ebrow2_col3\" class=\"data row2 col3\" >34.7</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_afd8a6c0_0fff_11eb_a306_b8f6b11350eblevel0_row3\" class=\"row_heading level0 row3\" >3</th>\n",
       "                        <td id=\"T_afd8a6c0_0fff_11eb_a306_b8f6b11350ebrow3_col0\" class=\"data row3 col0\" >6.998</td>\n",
       "                        <td id=\"T_afd8a6c0_0fff_11eb_a306_b8f6b11350ebrow3_col1\" class=\"data row3 col1\" >2.94</td>\n",
       "                        <td id=\"T_afd8a6c0_0fff_11eb_a306_b8f6b11350ebrow3_col2\" class=\"data row3 col2\" >18.7</td>\n",
       "                        <td id=\"T_afd8a6c0_0fff_11eb_a306_b8f6b11350ebrow3_col3\" class=\"data row3 col3\" >33.4</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_afd8a6c0_0fff_11eb_a306_b8f6b11350eblevel0_row4\" class=\"row_heading level0 row4\" >4</th>\n",
       "                        <td id=\"T_afd8a6c0_0fff_11eb_a306_b8f6b11350ebrow4_col0\" class=\"data row4 col0\" >7.147</td>\n",
       "                        <td id=\"T_afd8a6c0_0fff_11eb_a306_b8f6b11350ebrow4_col1\" class=\"data row4 col1\" >5.33</td>\n",
       "                        <td id=\"T_afd8a6c0_0fff_11eb_a306_b8f6b11350ebrow4_col2\" class=\"data row4 col2\" >18.7</td>\n",
       "                        <td id=\"T_afd8a6c0_0fff_11eb_a306_b8f6b11350ebrow4_col3\" class=\"data row4 col3\" >36.2</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_afd8a6c0_0fff_11eb_a306_b8f6b11350eblevel0_row5\" class=\"row_heading level0 row5\" >5</th>\n",
       "                        <td id=\"T_afd8a6c0_0fff_11eb_a306_b8f6b11350ebrow5_col0\" class=\"data row5 col0\" >6.43</td>\n",
       "                        <td id=\"T_afd8a6c0_0fff_11eb_a306_b8f6b11350ebrow5_col1\" class=\"data row5 col1\" >5.21</td>\n",
       "                        <td id=\"T_afd8a6c0_0fff_11eb_a306_b8f6b11350ebrow5_col2\" class=\"data row5 col2\" >18.7</td>\n",
       "                        <td id=\"T_afd8a6c0_0fff_11eb_a306_b8f6b11350ebrow5_col3\" class=\"data row5 col3\" >28.7</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_afd8a6c0_0fff_11eb_a306_b8f6b11350eblevel0_row6\" class=\"row_heading level0 row6\" >6</th>\n",
       "                        <td id=\"T_afd8a6c0_0fff_11eb_a306_b8f6b11350ebrow6_col0\" class=\"data row6 col0\" >6.012</td>\n",
       "                        <td id=\"T_afd8a6c0_0fff_11eb_a306_b8f6b11350ebrow6_col1\" class=\"data row6 col1\" >12.43</td>\n",
       "                        <td id=\"T_afd8a6c0_0fff_11eb_a306_b8f6b11350ebrow6_col2\" class=\"data row6 col2\" >15.2</td>\n",
       "                        <td id=\"T_afd8a6c0_0fff_11eb_a306_b8f6b11350ebrow6_col3\" class=\"data row6 col3\" >22.9</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_afd8a6c0_0fff_11eb_a306_b8f6b11350eblevel0_row7\" class=\"row_heading level0 row7\" >7</th>\n",
       "                        <td id=\"T_afd8a6c0_0fff_11eb_a306_b8f6b11350ebrow7_col0\" class=\"data row7 col0\" >6.172</td>\n",
       "                        <td id=\"T_afd8a6c0_0fff_11eb_a306_b8f6b11350ebrow7_col1\" class=\"data row7 col1\" >19.15</td>\n",
       "                        <td id=\"T_afd8a6c0_0fff_11eb_a306_b8f6b11350ebrow7_col2\" class=\"data row7 col2\" >15.2</td>\n",
       "                        <td id=\"T_afd8a6c0_0fff_11eb_a306_b8f6b11350ebrow7_col3\" class=\"data row7 col3\" >27.1</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_afd8a6c0_0fff_11eb_a306_b8f6b11350eblevel0_row8\" class=\"row_heading level0 row8\" >8</th>\n",
       "                        <td id=\"T_afd8a6c0_0fff_11eb_a306_b8f6b11350ebrow8_col0\" class=\"data row8 col0\" >5.631</td>\n",
       "                        <td id=\"T_afd8a6c0_0fff_11eb_a306_b8f6b11350ebrow8_col1\" class=\"data row8 col1\" >29.93</td>\n",
       "                        <td id=\"T_afd8a6c0_0fff_11eb_a306_b8f6b11350ebrow8_col2\" class=\"data row8 col2\" >15.2</td>\n",
       "                        <td id=\"T_afd8a6c0_0fff_11eb_a306_b8f6b11350ebrow8_col3\" class=\"data row8 col3\" >16.5</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_afd8a6c0_0fff_11eb_a306_b8f6b11350eblevel0_row9\" class=\"row_heading level0 row9\" >9</th>\n",
       "                        <td id=\"T_afd8a6c0_0fff_11eb_a306_b8f6b11350ebrow9_col0\" class=\"data row9 col0\" >6.004</td>\n",
       "                        <td id=\"T_afd8a6c0_0fff_11eb_a306_b8f6b11350ebrow9_col1\" class=\"data row9 col1\" >17.1</td>\n",
       "                        <td id=\"T_afd8a6c0_0fff_11eb_a306_b8f6b11350ebrow9_col2\" class=\"data row9 col2\" >15.2</td>\n",
       "                        <td id=\"T_afd8a6c0_0fff_11eb_a306_b8f6b11350ebrow9_col3\" class=\"data row9 col3\" >18.9</td>\n",
       "            </tr>\n",
       "    </tbody></table>"
      ],
      "text/plain": [
       "<pandas.io.formats.style.Styler at 0x116854f90>"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "\n",
    "def highlight_color(s, fullDFsplit):\n",
    "    '''\n",
    "    highlight the the entire dataframe cyan.\n",
    "    '''\n",
    "\n",
    "    colorDF = s.copy()\n",
    "\n",
    "\n",
    "    colorDF.loc[fullDFsplit['split'] == 'train', ['RM', 'LSTAT', 'PTRATIO']] = 'background-color: #40E0D0'\n",
    "\n",
    "\n",
    "    colorDF.loc[fullDFsplit['split'] == 'test', ['RM', 'LSTAT', 'PTRATIO']] = 'background-color: #00FFFF'\n",
    "\n",
    "    # #9370DB\n",
    "    # FF D7 00\n",
    "    colorDF.loc[fullDFsplit['split'] == 'train', ['target']] = 'background-color: #FFD700'\n",
    "\n",
    "    # EE82EE\n",
    "    # BD B7 6B\n",
    "    colorDF.loc[fullDFsplit['split'] == 'test', ['target']] = 'background-color: #FFFF00'\n",
    "    return(colorDF)\n",
    "\n",
    "\n",
    "temp = fullDF.sort_index().loc[0:9,:].style.apply(lambda x: highlight_color(x,pd.DataFrame(fullDFsplit['split'])), axis = None)\n",
    "temp.set_properties(**{'border-color': 'black',\n",
    "                       'border': '1px solid black'})"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h3>Train test split key</h3>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>X_train</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>X_test</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>y_train</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>y_test</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         0\n",
       "0  X_train\n",
       "1   X_test\n",
       "2  y_train\n",
       "3   y_test"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Train test split key\n",
    "temp = pd.DataFrame(data = [['X_train','X_test','y_train','y_test']]).T\n",
    "temp"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<style  type=\"text/css\" >\n",
       "    #T_b0f36392_0fff_11eb_a306_b8f6b11350ebrow0_col0 {\n",
       "            background-color:  #40E0D0;\n",
       "            border-color:  black;\n",
       "            border:  1px solid black;\n",
       "        }    #T_b0f36392_0fff_11eb_a306_b8f6b11350ebrow1_col0 {\n",
       "            background-color:  #00FFFF;\n",
       "            border-color:  black;\n",
       "            border:  1px solid black;\n",
       "        }    #T_b0f36392_0fff_11eb_a306_b8f6b11350ebrow2_col0 {\n",
       "            background-color:  #FFD700;\n",
       "            border-color:  black;\n",
       "            border:  1px solid black;\n",
       "        }    #T_b0f36392_0fff_11eb_a306_b8f6b11350ebrow3_col0 {\n",
       "            background-color:  #FFFF00;\n",
       "            border-color:  black;\n",
       "            border:  1px solid black;\n",
       "        }</style><table id=\"T_b0f36392_0fff_11eb_a306_b8f6b11350eb\" ><thead>    <tr>        <th class=\"blank level0\" ></th>        <th class=\"col_heading level0 col0\" >0</th>    </tr></thead><tbody>\n",
       "                <tr>\n",
       "                        <th id=\"T_b0f36392_0fff_11eb_a306_b8f6b11350eblevel0_row0\" class=\"row_heading level0 row0\" >0</th>\n",
       "                        <td id=\"T_b0f36392_0fff_11eb_a306_b8f6b11350ebrow0_col0\" class=\"data row0 col0\" >X_train</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_b0f36392_0fff_11eb_a306_b8f6b11350eblevel0_row1\" class=\"row_heading level0 row1\" >1</th>\n",
       "                        <td id=\"T_b0f36392_0fff_11eb_a306_b8f6b11350ebrow1_col0\" class=\"data row1 col0\" >X_test</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_b0f36392_0fff_11eb_a306_b8f6b11350eblevel0_row2\" class=\"row_heading level0 row2\" >2</th>\n",
       "                        <td id=\"T_b0f36392_0fff_11eb_a306_b8f6b11350ebrow2_col0\" class=\"data row2 col0\" >y_train</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_b0f36392_0fff_11eb_a306_b8f6b11350eblevel0_row3\" class=\"row_heading level0 row3\" >3</th>\n",
       "                        <td id=\"T_b0f36392_0fff_11eb_a306_b8f6b11350ebrow3_col0\" class=\"data row3 col0\" >y_test</td>\n",
       "            </tr>\n",
       "    </tbody></table>"
      ],
      "text/plain": [
       "<pandas.io.formats.style.Styler at 0x1a1abe4e50>"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def highlight_mini(s):\n",
    "    '''\n",
    "    highlight the the entire dataframe cyan.\n",
    "    '''\n",
    "\n",
    "    colorDF = s.copy()\n",
    "\n",
    "    # colorDF.loc[0, [0]] = 'background-color: #40E0D0'\n",
    "    \n",
    "    # train features\n",
    "    colorDF.loc[0, [0]] = 'background-color: #40E0D0'\n",
    "\n",
    "    # test features\n",
    "    colorDF.loc[1, [0]] = 'background-color: #00FFFF'\n",
    "\n",
    "    # train target\n",
    "    colorDF.loc[2, [0]] = 'background-color: #FFD700'\n",
    "\n",
    "    # test target\n",
    "    colorDF.loc[3, [0]] = 'background-color: #FFFF00'\n",
    "\n",
    "    return(colorDF)\n",
    "\n",
    "\n",
    "temp2 = temp.sort_index().style.apply(lambda x: highlight_mini(x), axis = None)\n",
    "temp2.set_properties(**{'border-color': 'black',\n",
    "                       'border': '1px solid black',\n",
    "                       })"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "After that I was lazy and used powerpoint to make that graph. "
   ]
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}