ActiveLearningUCI_NOx.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Done uploading libraries\n"
     ]
    }
   ],
   "source": [
    "from two_TrAdaBoostR2 import TwoStageTrAdaBoostR2\n",
    "\n",
    "import pandas as pd\n",
    "import sys\n",
    "import numpy as np\n",
    "from pandas import DataFrame\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.tree import DecisionTreeRegressor\n",
    "from sklearn.ensemble import AdaBoostRegressor\n",
    "\n",
    "from keras.models import Sequential, load_model, Model\n",
    "from keras.layers import Input, Dense, Activation, Conv2D, Dropout, Flatten\n",
    "from keras import optimizers, utils, initializers, regularizers\n",
    "import keras.backend as K\n",
    "\n",
    "from sklearn.metrics import mean_squared_error\n",
    "from sklearn.metrics import r2_score\n",
    "from sklearn.preprocessing import StandardScaler #Importing the StandardScaler\n",
    "\n",
    "from itertools import combinations\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "\n",
    "from scipy.stats.stats import pearsonr\n",
    "from math import sqrt\n",
    "\n",
    "################# Geo plotting libraries\n",
    "import geopandas as gdp\n",
    "from matplotlib.colors import ListedColormap\n",
    "# import geoplot as glpt\n",
    "\n",
    "import xgboost as xgb\n",
    "from sklearn.linear_model import LinearRegression\n",
    "from sklearn.linear_model import Ridge\n",
    "from sklearn import linear_model\n",
    "from keras.models import Sequential\n",
    "from keras.callbacks import ModelCheckpoint\n",
    "from sklearn.ensemble import GradientBoostingRegressor\n",
    "\n",
    "from statistics import mean\n",
    "from sklearn.cluster import KMeans\n",
    "from scipy.spatial import distance\n",
    "\n",
    "print(\"Done uploading libraries\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['Date', 'Time', 'CO_GT', 'PT08_S1_CO', 'NMHC_GT', 'C6H6_GT',\n",
       "       'PT08_S2_NMHC', 'NOx_GT', 'PT08_S3_NOx', 'NO2_GT', 'PT08_S4_NO2',\n",
       "       'PT08_S5_O3', 'T', 'RH', 'AH', 'Unnamed: 15', 'Unnamed: 16'],\n",
       "      dtype='object')"
      ]
     },
     "execution_count": 47,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "############################## UCI Italian dataset #######################################################\n",
    "## 2 years and single terrain\n",
    "aqi_df = pd.read_csv('AQI_datasets/UCI_AQI/AirQualityUCI.csv', sep=',', delimiter=\";\", decimal=\",\", index_col = None, header=0)\n",
    "\n",
    "aqi_df.head(20)\n",
    "# aqi_df.shape\n",
    "aqi_df.columns\n",
    "\n",
    "## Predictors: T,Ah,Rh, NMHC_GT, NOx_GT, CO_GT, C6H6_GT ,Target: O3 "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>CO_GT</th>\n",
       "      <th>NMHC_GT</th>\n",
       "      <th>C6H6_GT</th>\n",
       "      <th>NOx_GT</th>\n",
       "      <th>NO2_GT</th>\n",
       "      <th>T</th>\n",
       "      <th>RH</th>\n",
       "      <th>AH</th>\n",
       "      <th>Year</th>\n",
       "      <th>Month</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2.6</td>\n",
       "      <td>150.0</td>\n",
       "      <td>11.9</td>\n",
       "      <td>166.0</td>\n",
       "      <td>113.0</td>\n",
       "      <td>13.6</td>\n",
       "      <td>48.9</td>\n",
       "      <td>0.7578</td>\n",
       "      <td>2004</td>\n",
       "      <td>10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2.0</td>\n",
       "      <td>112.0</td>\n",
       "      <td>9.4</td>\n",
       "      <td>103.0</td>\n",
       "      <td>92.0</td>\n",
       "      <td>13.3</td>\n",
       "      <td>47.7</td>\n",
       "      <td>0.7255</td>\n",
       "      <td>2004</td>\n",
       "      <td>10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2.2</td>\n",
       "      <td>88.0</td>\n",
       "      <td>9.0</td>\n",
       "      <td>131.0</td>\n",
       "      <td>114.0</td>\n",
       "      <td>11.9</td>\n",
       "      <td>54.0</td>\n",
       "      <td>0.7502</td>\n",
       "      <td>2004</td>\n",
       "      <td>10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>2.2</td>\n",
       "      <td>80.0</td>\n",
       "      <td>9.2</td>\n",
       "      <td>172.0</td>\n",
       "      <td>122.0</td>\n",
       "      <td>11.0</td>\n",
       "      <td>60.0</td>\n",
       "      <td>0.7867</td>\n",
       "      <td>2004</td>\n",
       "      <td>10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1.6</td>\n",
       "      <td>51.0</td>\n",
       "      <td>6.5</td>\n",
       "      <td>131.0</td>\n",
       "      <td>116.0</td>\n",
       "      <td>11.2</td>\n",
       "      <td>59.6</td>\n",
       "      <td>0.7888</td>\n",
       "      <td>2004</td>\n",
       "      <td>10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9352</th>\n",
       "      <td>3.1</td>\n",
       "      <td>275.0</td>\n",
       "      <td>13.5</td>\n",
       "      <td>472.0</td>\n",
       "      <td>190.0</td>\n",
       "      <td>21.9</td>\n",
       "      <td>29.3</td>\n",
       "      <td>0.7568</td>\n",
       "      <td>2005</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9353</th>\n",
       "      <td>2.4</td>\n",
       "      <td>275.0</td>\n",
       "      <td>11.4</td>\n",
       "      <td>353.0</td>\n",
       "      <td>179.0</td>\n",
       "      <td>24.3</td>\n",
       "      <td>23.7</td>\n",
       "      <td>0.7119</td>\n",
       "      <td>2005</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9354</th>\n",
       "      <td>2.4</td>\n",
       "      <td>275.0</td>\n",
       "      <td>12.4</td>\n",
       "      <td>293.0</td>\n",
       "      <td>175.0</td>\n",
       "      <td>26.9</td>\n",
       "      <td>18.3</td>\n",
       "      <td>0.6406</td>\n",
       "      <td>2005</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9355</th>\n",
       "      <td>2.1</td>\n",
       "      <td>275.0</td>\n",
       "      <td>9.5</td>\n",
       "      <td>235.0</td>\n",
       "      <td>156.0</td>\n",
       "      <td>28.3</td>\n",
       "      <td>13.5</td>\n",
       "      <td>0.5139</td>\n",
       "      <td>2005</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9356</th>\n",
       "      <td>2.2</td>\n",
       "      <td>275.0</td>\n",
       "      <td>11.9</td>\n",
       "      <td>265.0</td>\n",
       "      <td>168.0</td>\n",
       "      <td>28.5</td>\n",
       "      <td>13.1</td>\n",
       "      <td>0.5028</td>\n",
       "      <td>2005</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>9357 rows × 10 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "      CO_GT  NMHC_GT  C6H6_GT  NOx_GT  NO2_GT     T    RH      AH  Year  Month\n",
       "0       2.6    150.0     11.9   166.0   113.0  13.6  48.9  0.7578  2004     10\n",
       "1       2.0    112.0      9.4   103.0    92.0  13.3  47.7  0.7255  2004     10\n",
       "2       2.2     88.0      9.0   131.0   114.0  11.9  54.0  0.7502  2004     10\n",
       "3       2.2     80.0      9.2   172.0   122.0  11.0  60.0  0.7867  2004     10\n",
       "4       1.6     51.0      6.5   131.0   116.0  11.2  59.6  0.7888  2004     10\n",
       "...     ...      ...      ...     ...     ...   ...   ...     ...   ...    ...\n",
       "9352    3.1    275.0     13.5   472.0   190.0  21.9  29.3  0.7568  2005      4\n",
       "9353    2.4    275.0     11.4   353.0   179.0  24.3  23.7  0.7119  2005      4\n",
       "9354    2.4    275.0     12.4   293.0   175.0  26.9  18.3  0.6406  2005      4\n",
       "9355    2.1    275.0      9.5   235.0   156.0  28.3  13.5  0.5139  2005      4\n",
       "9356    2.2    275.0     11.9   265.0   168.0  28.5  13.1  0.5028  2005      4\n",
       "\n",
       "[9357 rows x 10 columns]"
      ]
     },
     "execution_count": 48,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def remove_outlier(col):\n",
    "    aqi_df[col] = aqi_df.groupby('Date')[col].transform(lambda x: x.fillna(x.mean()))\n",
    "\n",
    "\n",
    "#drop end rows with NaN values\n",
    "aqi_df.dropna(how = 'all', inplace = True)\n",
    "\n",
    "drop_unamed = ['Unnamed: 15', 'Unnamed: 16']\n",
    "aqi_df = aqi_df.drop(drop_unamed, axis = 1)\n",
    "aqi_df.columns\n",
    "# aqi_df.shape\n",
    "# aqi_df\n",
    "\n",
    "#Observing data statistics\n",
    "aqi_df.describe()\n",
    "\n",
    "#Drop unwanted columns\n",
    "drop_uw = ['Time', 'PT08_S1_CO', 'PT08_S2_NMHC', 'PT08_S5_O3', 'PT08_S3_NOx', 'PT08_S4_NO2']\n",
    "aqi_df = aqi_df.drop(drop_uw, axis = 1)\n",
    "aqi_df\n",
    "\n",
    "#Replace the -200 values seen in the dataset with \n",
    "aqi_df.replace(to_replace = -200, value = np.NaN, inplace = True)\n",
    "\n",
    "#Replace the NaN values with the column mean\n",
    "col_list = aqi_df.columns[1:]\n",
    "for i in col_list:\n",
    "    remove_outlier(i)\n",
    "\n",
    "aqi_df.fillna(method ='ffill', inplace= True)\n",
    "aqi_df.dropna(axis = 0)\n",
    "aqi_df\n",
    "\n",
    "#Convert 'Date' column to datetime and then seperate out year and month into different columns.\n",
    "aqi_df.Date = pd.to_datetime(aqi_df.Date)\n",
    "aqi_df['Year'] = aqi_df['Date'].dt.year\n",
    "aqi_df['Month'] = aqi_df['Date'].dt.month\n",
    "drop_date = ['Date']\n",
    "aqi_df = aqi_df.drop(drop_date, axis = 1)\n",
    "aqi_df\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>CO</th>\n",
       "      <th>NMHC</th>\n",
       "      <th>C6H6</th>\n",
       "      <th>NO2</th>\n",
       "      <th>Temp</th>\n",
       "      <th>RH</th>\n",
       "      <th>AH</th>\n",
       "      <th>Year</th>\n",
       "      <th>Month</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2.134783</td>\n",
       "      <td>275.0</td>\n",
       "      <td>4.2</td>\n",
       "      <td>128.409091</td>\n",
       "      <td>8.2</td>\n",
       "      <td>40.1</td>\n",
       "      <td>0.4375</td>\n",
       "      <td>2005</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1.600000</td>\n",
       "      <td>275.0</td>\n",
       "      <td>8.8</td>\n",
       "      <td>106.000000</td>\n",
       "      <td>5.3</td>\n",
       "      <td>50.7</td>\n",
       "      <td>0.4564</td>\n",
       "      <td>2005</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2.500000</td>\n",
       "      <td>275.0</td>\n",
       "      <td>7.5</td>\n",
       "      <td>129.000000</td>\n",
       "      <td>5.9</td>\n",
       "      <td>50.0</td>\n",
       "      <td>0.4689</td>\n",
       "      <td>2005</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>2.700000</td>\n",
       "      <td>275.0</td>\n",
       "      <td>7.6</td>\n",
       "      <td>128.409091</td>\n",
       "      <td>4.9</td>\n",
       "      <td>53.9</td>\n",
       "      <td>0.4693</td>\n",
       "      <td>2005</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1.900000</td>\n",
       "      <td>275.0</td>\n",
       "      <td>5.6</td>\n",
       "      <td>126.000000</td>\n",
       "      <td>4.3</td>\n",
       "      <td>55.3</td>\n",
       "      <td>0.4650</td>\n",
       "      <td>2005</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2242</th>\n",
       "      <td>3.100000</td>\n",
       "      <td>275.0</td>\n",
       "      <td>13.5</td>\n",
       "      <td>190.000000</td>\n",
       "      <td>21.9</td>\n",
       "      <td>29.3</td>\n",
       "      <td>0.7568</td>\n",
       "      <td>2005</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2243</th>\n",
       "      <td>2.400000</td>\n",
       "      <td>275.0</td>\n",
       "      <td>11.4</td>\n",
       "      <td>179.000000</td>\n",
       "      <td>24.3</td>\n",
       "      <td>23.7</td>\n",
       "      <td>0.7119</td>\n",
       "      <td>2005</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2244</th>\n",
       "      <td>2.400000</td>\n",
       "      <td>275.0</td>\n",
       "      <td>12.4</td>\n",
       "      <td>175.000000</td>\n",
       "      <td>26.9</td>\n",
       "      <td>18.3</td>\n",
       "      <td>0.6406</td>\n",
       "      <td>2005</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2245</th>\n",
       "      <td>2.100000</td>\n",
       "      <td>275.0</td>\n",
       "      <td>9.5</td>\n",
       "      <td>156.000000</td>\n",
       "      <td>28.3</td>\n",
       "      <td>13.5</td>\n",
       "      <td>0.5139</td>\n",
       "      <td>2005</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2246</th>\n",
       "      <td>2.200000</td>\n",
       "      <td>275.0</td>\n",
       "      <td>11.9</td>\n",
       "      <td>168.000000</td>\n",
       "      <td>28.5</td>\n",
       "      <td>13.1</td>\n",
       "      <td>0.5028</td>\n",
       "      <td>2005</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>2247 rows × 9 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "            CO   NMHC  C6H6         NO2  Temp    RH      AH  Year  Month\n",
       "0     2.134783  275.0   4.2  128.409091   8.2  40.1  0.4375  2005      1\n",
       "1     1.600000  275.0   8.8  106.000000   5.3  50.7  0.4564  2005      1\n",
       "2     2.500000  275.0   7.5  129.000000   5.9  50.0  0.4689  2005      1\n",
       "3     2.700000  275.0   7.6  128.409091   4.9  53.9  0.4693  2005      1\n",
       "4     1.900000  275.0   5.6  126.000000   4.3  55.3  0.4650  2005      1\n",
       "...        ...    ...   ...         ...   ...   ...     ...   ...    ...\n",
       "2242  3.100000  275.0  13.5  190.000000  21.9  29.3  0.7568  2005      4\n",
       "2243  2.400000  275.0  11.4  179.000000  24.3  23.7  0.7119  2005      4\n",
       "2244  2.400000  275.0  12.4  175.000000  26.9  18.3  0.6406  2005      4\n",
       "2245  2.100000  275.0   9.5  156.000000  28.3  13.5  0.5139  2005      4\n",
       "2246  2.200000  275.0  11.9  168.000000  28.5  13.1  0.5028  2005      4\n",
       "\n",
       "[2247 rows x 9 columns]"
      ]
     },
     "execution_count": 49,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#Split the dataset according to the year.\n",
    "aqi_df_2004 = aqi_df[aqi_df['Year'] == 2004]\n",
    "aqi_df_2005 = aqi_df[aqi_df['Year'] == 2005]\n",
    "\n",
    "aqi_df_2004 = aqi_df_2004.reset_index(drop=True)\n",
    "aqi_df_2005 = aqi_df_2005.reset_index(drop=True)\n",
    "\n",
    "\n",
    "#Divide the dataframe into target and the predictors.\n",
    "target_uci_col = ['NOx_GT']\n",
    "aqi_df_2004_target = aqi_df_2004[target_uci_col]\n",
    "aqi_df_2004_target.columns = ['NOx']\n",
    "\n",
    "aqi_df_2005_target = aqi_df_2005[target_uci_col]\n",
    "aqi_df_2005_target.columns = ['NOx']\n",
    "\n",
    "aqi_df_2004_predictors = aqi_df_2004.drop(target_uci_col, axis = 1)\n",
    "aqi_df_2004_predictors.columns = ['CO', 'NMHC', 'C6H6', 'NO2', 'Temp', 'RH', 'AH', 'Year', 'Month']\n",
    "\n",
    "aqi_df_2005_predictors = aqi_df_2005.drop(target_uci_col, axis = 1)\n",
    "aqi_df_2005_predictors.columns = ['CO', 'NMHC', 'C6H6', 'NO2', 'Temp', 'RH', 'AH', 'Year', 'Month']\n",
    "\n",
    "aqi_df_2004_target\n",
    "aqi_df_2004_predictors\n",
    "\n",
    "aqi_df_2005_target\n",
    "aqi_df_2005_predictors\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>CO</th>\n",
       "      <th>NMHC</th>\n",
       "      <th>C6H6</th>\n",
       "      <th>NO2</th>\n",
       "      <th>Temp</th>\n",
       "      <th>RH</th>\n",
       "      <th>AH</th>\n",
       "      <th>Year</th>\n",
       "      <th>Month</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.357434</td>\n",
       "      <td>-1.403072</td>\n",
       "      <td>0.167727</td>\n",
       "      <td>0.358847</td>\n",
       "      <td>-0.897718</td>\n",
       "      <td>0.030505</td>\n",
       "      <td>-1.028223</td>\n",
       "      <td>2004</td>\n",
       "      <td>10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>-0.073637</td>\n",
       "      <td>-1.872833</td>\n",
       "      <td>-0.163670</td>\n",
       "      <td>-0.177882</td>\n",
       "      <td>-0.935168</td>\n",
       "      <td>-0.038507</td>\n",
       "      <td>-1.115824</td>\n",
       "      <td>2004</td>\n",
       "      <td>10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.070053</td>\n",
       "      <td>-2.169524</td>\n",
       "      <td>-0.216693</td>\n",
       "      <td>0.384406</td>\n",
       "      <td>-1.109937</td>\n",
       "      <td>0.323803</td>\n",
       "      <td>-1.048835</td>\n",
       "      <td>2004</td>\n",
       "      <td>10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.070053</td>\n",
       "      <td>-2.268421</td>\n",
       "      <td>-0.190181</td>\n",
       "      <td>0.588874</td>\n",
       "      <td>-1.222288</td>\n",
       "      <td>0.668860</td>\n",
       "      <td>-0.949843</td>\n",
       "      <td>2004</td>\n",
       "      <td>10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>-0.361018</td>\n",
       "      <td>-2.626923</td>\n",
       "      <td>-0.548090</td>\n",
       "      <td>0.435523</td>\n",
       "      <td>-1.197321</td>\n",
       "      <td>0.645856</td>\n",
       "      <td>-0.944148</td>\n",
       "      <td>2004</td>\n",
       "      <td>10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7105</th>\n",
       "      <td>-0.648398</td>\n",
       "      <td>0.142195</td>\n",
       "      <td>-0.707160</td>\n",
       "      <td>0.052145</td>\n",
       "      <td>-1.434508</td>\n",
       "      <td>-0.849390</td>\n",
       "      <td>-2.010003</td>\n",
       "      <td>2004</td>\n",
       "      <td>12</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7106</th>\n",
       "      <td>-0.648398</td>\n",
       "      <td>0.142195</td>\n",
       "      <td>-0.707160</td>\n",
       "      <td>0.052145</td>\n",
       "      <td>-1.397057</td>\n",
       "      <td>-0.924153</td>\n",
       "      <td>-2.033598</td>\n",
       "      <td>2004</td>\n",
       "      <td>12</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7107</th>\n",
       "      <td>-0.648398</td>\n",
       "      <td>0.142195</td>\n",
       "      <td>-0.680649</td>\n",
       "      <td>0.052145</td>\n",
       "      <td>-1.459475</td>\n",
       "      <td>-0.912651</td>\n",
       "      <td>-2.062075</td>\n",
       "      <td>2004</td>\n",
       "      <td>12</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7108</th>\n",
       "      <td>-0.648398</td>\n",
       "      <td>0.142195</td>\n",
       "      <td>-0.667393</td>\n",
       "      <td>0.052145</td>\n",
       "      <td>-1.459475</td>\n",
       "      <td>-0.642356</td>\n",
       "      <td>-1.914537</td>\n",
       "      <td>2004</td>\n",
       "      <td>12</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7109</th>\n",
       "      <td>-0.648398</td>\n",
       "      <td>0.142195</td>\n",
       "      <td>-0.799951</td>\n",
       "      <td>0.052145</td>\n",
       "      <td>-1.621760</td>\n",
       "      <td>-0.573345</td>\n",
       "      <td>-1.975559</td>\n",
       "      <td>2004</td>\n",
       "      <td>12</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>7110 rows × 9 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "            CO      NMHC      C6H6       NO2      Temp        RH        AH  \\\n",
       "0     0.357434 -1.403072  0.167727  0.358847 -0.897718  0.030505 -1.028223   \n",
       "1    -0.073637 -1.872833 -0.163670 -0.177882 -0.935168 -0.038507 -1.115824   \n",
       "2     0.070053 -2.169524 -0.216693  0.384406 -1.109937  0.323803 -1.048835   \n",
       "3     0.070053 -2.268421 -0.190181  0.588874 -1.222288  0.668860 -0.949843   \n",
       "4    -0.361018 -2.626923 -0.548090  0.435523 -1.197321  0.645856 -0.944148   \n",
       "...        ...       ...       ...       ...       ...       ...       ...   \n",
       "7105 -0.648398  0.142195 -0.707160  0.052145 -1.434508 -0.849390 -2.010003   \n",
       "7106 -0.648398  0.142195 -0.707160  0.052145 -1.397057 -0.924153 -2.033598   \n",
       "7107 -0.648398  0.142195 -0.680649  0.052145 -1.459475 -0.912651 -2.062075   \n",
       "7108 -0.648398  0.142195 -0.667393  0.052145 -1.459475 -0.642356 -1.914537   \n",
       "7109 -0.648398  0.142195 -0.799951  0.052145 -1.621760 -0.573345 -1.975559   \n",
       "\n",
       "      Year  Month  \n",
       "0     2004     10  \n",
       "1     2004     10  \n",
       "2     2004     10  \n",
       "3     2004     10  \n",
       "4     2004     10  \n",
       "...    ...    ...  \n",
       "7105  2004     12  \n",
       "7106  2004     12  \n",
       "7107  2004     12  \n",
       "7108  2004     12  \n",
       "7109  2004     12  \n",
       "\n",
       "[7110 rows x 9 columns]"
      ]
     },
     "execution_count": 50,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "## Standardize the dataset\n",
    "columns_uci = ['CO', 'NMHC', 'C6H6', 'NO2', 'Temp', 'RH', 'AH', 'Year', 'Month']\n",
    "cols_to_norm = ['CO', 'NMHC', 'C6H6', 'NO2', 'Temp', 'RH', 'AH']\n",
    "\n",
    "ss = StandardScaler()\n",
    "aqi_df_2004_predictors[cols_to_norm] = ss.fit_transform(aqi_df_2004_predictors[cols_to_norm])\n",
    "aqi_df_2005_predictors[cols_to_norm] = ss.fit_transform(aqi_df_2005_predictors[cols_to_norm])\n",
    "\n",
    "aqi_df_2004_predictors\n",
    "# aqi_df_2005_predictors"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "<ipython-input-51-bce9a2e6f612>:13: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  X_train['NOx'] = y_2005_train\n",
      "<ipython-input-51-bce9a2e6f612>:16: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  X_test['NOx'] = y_2005_test\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(            CO      NMHC      C6H6       NO2      Temp        RH        AH  \\\n",
       " 0     0.357434 -1.403072  0.167727  0.358847 -0.897718  0.030505 -1.028223   \n",
       " 1    -0.073637 -1.872833 -0.163670 -0.177882 -0.935168 -0.038507 -1.115824   \n",
       " 2     0.070053 -2.169524 -0.216693  0.384406 -1.109937  0.323803 -1.048835   \n",
       " 3     0.070053 -2.268421 -0.190181  0.588874 -1.222288  0.668860 -0.949843   \n",
       " 4    -0.361018 -2.626923 -0.548090  0.435523 -1.197321  0.645856 -0.944148   \n",
       " ...        ...       ...       ...       ...       ...       ...       ...   \n",
       " 7105 -0.648398  0.142195 -0.707160  0.052145 -1.434508 -0.849390 -2.010003   \n",
       " 7106 -0.648398  0.142195 -0.707160  0.052145 -1.397057 -0.924153 -2.033598   \n",
       " 7107 -0.648398  0.142195 -0.680649  0.052145 -1.459475 -0.912651 -2.062075   \n",
       " 7108 -0.648398  0.142195 -0.667393  0.052145 -1.459475 -0.642356 -1.914537   \n",
       " 7109 -0.648398  0.142195 -0.799951  0.052145 -1.621760 -0.573345 -1.975559   \n",
       " \n",
       "       Year  Month    NOx  \n",
       " 0     2004     10  166.0  \n",
       " 1     2004     10  103.0  \n",
       " 2     2004     10  131.0  \n",
       " 3     2004     10  172.0  \n",
       " 4     2004     10  131.0  \n",
       " ...    ...    ...    ...  \n",
       " 7105  2004     12  165.0  \n",
       " 7106  2004     12  165.0  \n",
       " 7107  2004     12  165.0  \n",
       " 7108  2004     12  165.0  \n",
       " 7109  2004     12  165.0  \n",
       " \n",
       " [7110 rows x 10 columns],\n",
       " (89, 10),\n",
       " (2158, 10))"
      ]
     },
     "execution_count": 51,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#Splitting dataset into train, test and source.\n",
    "### Case 1: 0.96\n",
    "### Case 2: 0.999\n",
    "X_2005_train, X_2005_test, y_2005_train, y_2005_test = train_test_split(aqi_df_2005_predictors, aqi_df_2005_target, test_size = 0.96, random_state = 1)\n",
    "\n",
    "X_2004 = aqi_df_2004_predictors\n",
    "y_2004 = aqi_df_2004_target\n",
    "\n",
    "X_source = X_2004\n",
    "X_source['NOx'] = y_2004\n",
    "\n",
    "X_train = X_2005_train\n",
    "X_train['NOx'] = y_2005_train\n",
    "\n",
    "X_test = X_2005_test\n",
    "X_test['NOx'] = y_2005_test\n",
    "\n",
    "X_source, X_train.shape, X_test.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Row selected:  74\n",
      "Min. distance:  912.7715872062486\n",
      "Matrix shape:  (89, 10)\n",
      "Row selected:  20\n",
      "Min. distance:  772.3761604547398\n",
      "Matrix shape:  (88, 10)\n",
      "Row selected:  41\n",
      "Min. distance:  504.30971141391166\n",
      "Matrix shape:  (87, 10)\n",
      "Row selected:  66\n",
      "Min. distance:  559.7742067083276\n",
      "Matrix shape:  (86, 10)\n",
      "Row selected:  84\n",
      "Min. distance:  570.8319553379229\n",
      "Matrix shape:  (85, 10)\n",
      "Row selected:  67\n",
      "Min. distance:  1073.926068802342\n",
      "Matrix shape:  (84, 10)\n",
      "Row selected:  21\n",
      "Min. distance:  372.42016652848247\n",
      "Matrix shape:  (83, 10)\n",
      "Row selected:  73\n",
      "Min. distance:  407.154236932868\n",
      "Matrix shape:  (82, 10)\n",
      "Row selected:  51\n",
      "Min. distance:  509.9861664911643\n",
      "Matrix shape:  (81, 10)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(9, 10)"
      ]
     },
     "execution_count": 52,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "######################################## Seeding Technique ###################################################\n",
    "kmeans = KMeans(n_clusters = 9, random_state=0).fit(X_source)\n",
    "\n",
    "Alternate_df = X_train.copy()\n",
    "idxlist = []\n",
    "Alternate_df_np = Alternate_df.to_numpy()\n",
    "\n",
    "new_df_list = []\n",
    "\n",
    "\n",
    "for rowkm in kmeans.cluster_centers_:\n",
    "    mindist = -99\n",
    "    rowidx = 0\n",
    "    idx = 0\n",
    "    for row in Alternate_df_np:\n",
    "        dst = distance.euclidean(row, rowkm)\n",
    "\n",
    "        if(dst >= mindist):\n",
    "            mindist = dst\n",
    "            rowidx = idx\n",
    "            rowval = row\n",
    "\n",
    "        idx = idx + 1\n",
    "\n",
    "    print(\"Row selected: \", rowidx) #Alternate_df.loc[rowidx,:]\\\n",
    "    print(\"Min. distance: \", mindist)\n",
    "    print(\"Matrix shape: \", Alternate_df_np.shape)\n",
    "    new_df_list.append(rowval)\n",
    "    Alternate_df_np = np.delete(Alternate_df_np, rowidx, 0)\n",
    "    idxlist.append(rowidx)\n",
    "\n",
    "\n",
    "new_df = pd.DataFrame(np.vstack(new_df_list))\n",
    "#new_df\n",
    "new_df.shape\n",
    "#new_df.dtypes\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Row selected:  7109\n",
      "Min. distance:  801.0890782673067\n",
      "Matrix shape:  (7110, 10)\n",
      "Row selected:  6986\n",
      "Min. distance:  548.1285851288357\n",
      "Matrix shape:  (7109, 10)\n",
      "Row selected:  7107\n",
      "Min. distance:  698.0173642865282\n",
      "Matrix shape:  (7108, 10)\n",
      "Row selected:  7106\n",
      "Min. distance:  578.0232036782321\n",
      "Matrix shape:  (7107, 10)\n",
      "Row selected:  6962\n",
      "Min. distance:  767.0911383692626\n",
      "Matrix shape:  (7106, 10)\n",
      "Row selected:  6961\n",
      "Min. distance:  533.1280259309375\n",
      "Matrix shape:  (7105, 10)\n",
      "Row selected:  6993\n",
      "Min. distance:  548.1175871471462\n",
      "Matrix shape:  (7104, 10)\n",
      "Row selected:  6955\n",
      "Min. distance:  530.1236340422324\n",
      "Matrix shape:  (7103, 10)\n",
      "Row selected:  6991\n",
      "Min. distance:  578.1257502042014\n",
      "Matrix shape:  (7102, 10)\n",
      "Shape before : (7110, 10)\n",
      "Shape after : (7071, 10)\n"
     ]
    }
   ],
   "source": [
    "##################################################### Phase 2: Seeding ################################################\n",
    "\n",
    "alt_source_df = X_source.copy()\n",
    "idxlist2 = []\n",
    "alt_source_df_np = alt_source_df.to_numpy()\n",
    "\n",
    "final_df_list = []\n",
    "\n",
    "for row_nm in new_df_list:\n",
    "    min_dist = -99\n",
    "    row_idx = 0\n",
    "    idx_val = 0\n",
    "    for row_alt in alt_source_df_np:\n",
    "        dst = distance.euclidean(row_alt, row_nm)\n",
    "        if(dst >= mindist):\n",
    "            min_dist = dst\n",
    "            row_idx = idx_val\n",
    "            row_val = row_alt\n",
    "\n",
    "        idx_val = idx_val + 1\n",
    "\n",
    "    print(\"Row selected: \", row_idx) #Alternate_df.loc[rowidx,:]\\\n",
    "    print(\"Min. distance: \", min_dist)\n",
    "    print(\"Matrix shape: \", alt_source_df_np.shape)\n",
    "    final_df_list.append(row_val)\n",
    "    alt_source_df_np = np.delete(alt_source_df_np, row_idx, 0)\n",
    "    idxlist2.append(row_idx)\n",
    "\n",
    "\n",
    "final_df = pd.DataFrame(np.vstack(final_df_list), columns= X_source.columns)\n",
    "final_df\n",
    "final_df.shape\n",
    "\n",
    "print(\"Shape before :\",X_source.shape)\n",
    "X_source = pd.concat([X_source, final_df])\n",
    "X_source = X_source.drop_duplicates(keep=False)\n",
    "print(\"Shape after :\",X_source.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>CO</th>\n",
       "      <th>NMHC</th>\n",
       "      <th>C6H6</th>\n",
       "      <th>NO2</th>\n",
       "      <th>Temp</th>\n",
       "      <th>RH</th>\n",
       "      <th>AH</th>\n",
       "      <th>Year</th>\n",
       "      <th>Month</th>\n",
       "      <th>NOx</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>-0.236089</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>-0.093770</td>\n",
       "      <td>-0.857417</td>\n",
       "      <td>-0.692434</td>\n",
       "      <td>1.618722</td>\n",
       "      <td>0.377326</td>\n",
       "      <td>2005.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>284.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>-0.753174</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>-0.963184</td>\n",
       "      <td>-0.995832</td>\n",
       "      <td>-0.961145</td>\n",
       "      <td>-0.867393</td>\n",
       "      <td>-1.269811</td>\n",
       "      <td>2005.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>104.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>-0.679304</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>-0.652679</td>\n",
       "      <td>-0.501494</td>\n",
       "      <td>1.564738</td>\n",
       "      <td>-1.575074</td>\n",
       "      <td>-0.319355</td>\n",
       "      <td>2005.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>161.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>-0.753174</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>-0.544002</td>\n",
       "      <td>-1.173794</td>\n",
       "      <td>0.024128</td>\n",
       "      <td>1.391033</td>\n",
       "      <td>1.089171</td>\n",
       "      <td>2005.0</td>\n",
       "      <td>12.0</td>\n",
       "      <td>183.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>-0.236089</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.828970</td>\n",
       "      <td>-0.264211</td>\n",
       "      <td>0.255453</td>\n",
       "      <td>0.841476</td>\n",
       "      <td>0.950868</td>\n",
       "      <td>2005.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>266.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>93</th>\n",
       "      <td>2.943861</td>\n",
       "      <td>0.142195</td>\n",
       "      <td>1.413779</td>\n",
       "      <td>1.917919</td>\n",
       "      <td>-1.184838</td>\n",
       "      <td>1.721283</td>\n",
       "      <td>-0.247952</td>\n",
       "      <td>2004.0</td>\n",
       "      <td>12.0</td>\n",
       "      <td>821.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>94</th>\n",
       "      <td>1.938028</td>\n",
       "      <td>0.142195</td>\n",
       "      <td>0.976335</td>\n",
       "      <td>1.815685</td>\n",
       "      <td>-1.234772</td>\n",
       "      <td>1.583260</td>\n",
       "      <td>-0.407153</td>\n",
       "      <td>2004.0</td>\n",
       "      <td>12.0</td>\n",
       "      <td>592.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>95</th>\n",
       "      <td>-1.151315</td>\n",
       "      <td>0.142195</td>\n",
       "      <td>-1.224139</td>\n",
       "      <td>-0.816846</td>\n",
       "      <td>-1.035036</td>\n",
       "      <td>0.513584</td>\n",
       "      <td>-0.840817</td>\n",
       "      <td>2004.0</td>\n",
       "      <td>12.0</td>\n",
       "      <td>93.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>96</th>\n",
       "      <td>2.440945</td>\n",
       "      <td>0.142195</td>\n",
       "      <td>0.989591</td>\n",
       "      <td>2.020153</td>\n",
       "      <td>-1.172354</td>\n",
       "      <td>0.835637</td>\n",
       "      <td>-0.790643</td>\n",
       "      <td>2004.0</td>\n",
       "      <td>12.0</td>\n",
       "      <td>602.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>97</th>\n",
       "      <td>-1.223160</td>\n",
       "      <td>0.142195</td>\n",
       "      <td>-1.316930</td>\n",
       "      <td>-1.353576</td>\n",
       "      <td>-1.109937</td>\n",
       "      <td>0.732120</td>\n",
       "      <td>-0.770845</td>\n",
       "      <td>2004.0</td>\n",
       "      <td>12.0</td>\n",
       "      <td>50.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>98 rows × 10 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "          CO      NMHC      C6H6       NO2      Temp        RH        AH  \\\n",
       "0  -0.236089  0.000000 -0.093770 -0.857417 -0.692434  1.618722  0.377326   \n",
       "1  -0.753174  0.000000 -0.963184 -0.995832 -0.961145 -0.867393 -1.269811   \n",
       "2  -0.679304  0.000000 -0.652679 -0.501494  1.564738 -1.575074 -0.319355   \n",
       "3  -0.753174  0.000000 -0.544002 -1.173794  0.024128  1.391033  1.089171   \n",
       "4  -0.236089  0.000000  0.828970 -0.264211  0.255453  0.841476  0.950868   \n",
       "..       ...       ...       ...       ...       ...       ...       ...   \n",
       "93  2.943861  0.142195  1.413779  1.917919 -1.184838  1.721283 -0.247952   \n",
       "94  1.938028  0.142195  0.976335  1.815685 -1.234772  1.583260 -0.407153   \n",
       "95 -1.151315  0.142195 -1.224139 -0.816846 -1.035036  0.513584 -0.840817   \n",
       "96  2.440945  0.142195  0.989591  2.020153 -1.172354  0.835637 -0.790643   \n",
       "97 -1.223160  0.142195 -1.316930 -1.353576 -1.109937  0.732120 -0.770845   \n",
       "\n",
       "      Year  Month    NOx  \n",
       "0   2005.0    7.0  284.0  \n",
       "1   2005.0    7.0  104.0  \n",
       "2   2005.0    2.0  161.0  \n",
       "3   2005.0   12.0  183.0  \n",
       "4   2005.0    5.0  266.0  \n",
       "..     ...    ...    ...  \n",
       "93  2004.0   12.0  821.0  \n",
       "94  2004.0   12.0  592.0  \n",
       "95  2004.0   12.0   93.0  \n",
       "96  2004.0   12.0  602.0  \n",
       "97  2004.0   12.0   50.0  \n",
       "\n",
       "[98 rows x 10 columns]"
      ]
     },
     "execution_count": 54,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_train = pd.concat([X_train, final_df], ignore_index=True)\n",
    "\n",
    "X_train.to_csv('ActiveSampling/O3_seeded_train.csv',index=False)\n",
    "X_source.to_csv('ActiveSampling/O3_seeded_source.csv',index=False)\n",
    "\n",
    "X_train\n",
    "# X_train.to_csv('ActiveSampling/UCI_NO2_activesampling_train.csv',index=False)\n",
    "# X_source.to_csv('ActiveSampling/UCI_NO2_activesampling_source.csv',index=False)\n",
    "# X_test.to_csv('ActiveSampling/UCI_NO2_activesampling_test.csv',index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Done !!!!!\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>CO</th>\n",
       "      <th>NMHC</th>\n",
       "      <th>C6H6</th>\n",
       "      <th>NO2</th>\n",
       "      <th>Temp</th>\n",
       "      <th>RH</th>\n",
       "      <th>AH</th>\n",
       "      <th>Year</th>\n",
       "      <th>Month</th>\n",
       "      <th>NOx</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>-0.236089</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>-0.093770</td>\n",
       "      <td>-0.857417</td>\n",
       "      <td>-0.692434</td>\n",
       "      <td>1.618722</td>\n",
       "      <td>0.377326</td>\n",
       "      <td>2005.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>284.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>-0.753174</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>-0.963184</td>\n",
       "      <td>-0.995832</td>\n",
       "      <td>-0.961145</td>\n",
       "      <td>-0.867393</td>\n",
       "      <td>-1.269811</td>\n",
       "      <td>2005.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>104.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>-0.679304</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>-0.652679</td>\n",
       "      <td>-0.501494</td>\n",
       "      <td>1.564738</td>\n",
       "      <td>-1.575074</td>\n",
       "      <td>-0.319355</td>\n",
       "      <td>2005.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>161.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>-0.753174</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>-0.544002</td>\n",
       "      <td>-1.173794</td>\n",
       "      <td>0.024128</td>\n",
       "      <td>1.391033</td>\n",
       "      <td>1.089171</td>\n",
       "      <td>2005.0</td>\n",
       "      <td>12.0</td>\n",
       "      <td>183.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>-0.236089</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.828970</td>\n",
       "      <td>-0.264211</td>\n",
       "      <td>0.255453</td>\n",
       "      <td>0.841476</td>\n",
       "      <td>0.950868</td>\n",
       "      <td>2005.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>266.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>93</th>\n",
       "      <td>2.943861</td>\n",
       "      <td>0.142195</td>\n",
       "      <td>1.413779</td>\n",
       "      <td>1.917919</td>\n",
       "      <td>-1.184838</td>\n",
       "      <td>1.721283</td>\n",
       "      <td>-0.247952</td>\n",
       "      <td>2004.0</td>\n",
       "      <td>12.0</td>\n",
       "      <td>821.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>94</th>\n",
       "      <td>1.938028</td>\n",
       "      <td>0.142195</td>\n",
       "      <td>0.976335</td>\n",
       "      <td>1.815685</td>\n",
       "      <td>-1.234772</td>\n",
       "      <td>1.583260</td>\n",
       "      <td>-0.407153</td>\n",
       "      <td>2004.0</td>\n",
       "      <td>12.0</td>\n",
       "      <td>592.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>95</th>\n",
       "      <td>-1.151315</td>\n",
       "      <td>0.142195</td>\n",
       "      <td>-1.224139</td>\n",
       "      <td>-0.816846</td>\n",
       "      <td>-1.035036</td>\n",
       "      <td>0.513584</td>\n",
       "      <td>-0.840817</td>\n",
       "      <td>2004.0</td>\n",
       "      <td>12.0</td>\n",
       "      <td>93.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>96</th>\n",
       "      <td>2.440945</td>\n",
       "      <td>0.142195</td>\n",
       "      <td>0.989591</td>\n",
       "      <td>2.020153</td>\n",
       "      <td>-1.172354</td>\n",
       "      <td>0.835637</td>\n",
       "      <td>-0.790643</td>\n",
       "      <td>2004.0</td>\n",
       "      <td>12.0</td>\n",
       "      <td>602.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>97</th>\n",
       "      <td>-1.223160</td>\n",
       "      <td>0.142195</td>\n",
       "      <td>-1.316930</td>\n",
       "      <td>-1.353576</td>\n",
       "      <td>-1.109937</td>\n",
       "      <td>0.732120</td>\n",
       "      <td>-0.770845</td>\n",
       "      <td>2004.0</td>\n",
       "      <td>12.0</td>\n",
       "      <td>50.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>98 rows × 10 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "          CO      NMHC      C6H6       NO2      Temp        RH        AH  \\\n",
       "0  -0.236089  0.000000 -0.093770 -0.857417 -0.692434  1.618722  0.377326   \n",
       "1  -0.753174  0.000000 -0.963184 -0.995832 -0.961145 -0.867393 -1.269811   \n",
       "2  -0.679304  0.000000 -0.652679 -0.501494  1.564738 -1.575074 -0.319355   \n",
       "3  -0.753174  0.000000 -0.544002 -1.173794  0.024128  1.391033  1.089171   \n",
       "4  -0.236089  0.000000  0.828970 -0.264211  0.255453  0.841476  0.950868   \n",
       "..       ...       ...       ...       ...       ...       ...       ...   \n",
       "93  2.943861  0.142195  1.413779  1.917919 -1.184838  1.721283 -0.247952   \n",
       "94  1.938028  0.142195  0.976335  1.815685 -1.234772  1.583260 -0.407153   \n",
       "95 -1.151315  0.142195 -1.224139 -0.816846 -1.035036  0.513584 -0.840817   \n",
       "96  2.440945  0.142195  0.989591  2.020153 -1.172354  0.835637 -0.790643   \n",
       "97 -1.223160  0.142195 -1.316930 -1.353576 -1.109937  0.732120 -0.770845   \n",
       "\n",
       "      Year  Month    NOx  \n",
       "0   2005.0    7.0  284.0  \n",
       "1   2005.0    7.0  104.0  \n",
       "2   2005.0    2.0  161.0  \n",
       "3   2005.0   12.0  183.0  \n",
       "4   2005.0    5.0  266.0  \n",
       "..     ...    ...    ...  \n",
       "93  2004.0   12.0  821.0  \n",
       "94  2004.0   12.0  592.0  \n",
       "95  2004.0   12.0   93.0  \n",
       "96  2004.0   12.0  602.0  \n",
       "97  2004.0   12.0   50.0  \n",
       "\n",
       "[98 rows x 10 columns]"
      ]
     },
     "execution_count": 55,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_train = pd.read_csv('ActiveSampling/O3_seeded_train.csv')\n",
    "X_source = pd.read_csv('ActiveSampling/O3_seeded_source.csv')\n",
    "\n",
    "X_source['ManDis'] = 0.0\n",
    "\n",
    "train_df_mean = []\n",
    "prow = X_train.mean()\n",
    "train_df_mean = [prow.CO, prow.NMHC, prow.C6H6, prow.NO2, prow.Temp, prow.RH, prow.AH, prow.Year, prow.Month, prow.NOx]\n",
    "\n",
    "rowidx = 0\n",
    "for row in X_source.itertuples():\n",
    "    row_list =[row.CO, row.NMHC, row.C6H6, row.NO2, row.Temp, row.RH, row.AH, row.Year, row.Month, row.NOx]\n",
    "\n",
    "    man_dis = 0\n",
    "    for i in range(0, len(row_list)):\n",
    "        tempval = train_df_mean[i] - row_list[i]\n",
    "        man_dis = man_dis + abs(tempval)\n",
    "    \n",
    "    X_source.loc[rowidx,\"ManDis\"] = man_dis\n",
    "    # print(Source_df.loc[rowidx,\"ManDis\"])\n",
    "    rowidx = rowidx + 1\n",
    "\n",
    "\n",
    "X_source_rest = X_source.sort_values('ManDis')\n",
    "X_source = X_source.head(2000)\n",
    "X_source_rest = X_source.iloc[2000:]\n",
    "X_source_rest = X_source_rest.drop(['ManDis'], axis =1)\n",
    "X_source = X_source.drop(['ManDis'], axis =1)\n",
    "\n",
    "X_train.to_csv('ActiveSampling/UCI_O3_activesampling_train.csv',index=False)\n",
    "X_source.to_csv('ActiveSampling/UCI_O3_activesampling_source.csv',index=False)\n",
    "X_test.to_csv('ActiveSampling/UCI_O3_activesampling_test.csv',index=False)\n",
    "\n",
    "print(\"Done !!!!!\")\n",
    "X_source\n",
    "X_train"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}