credit_risk_ensemble.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import warnings\n",
    "warnings.filterwarnings('ignore')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "from pathlib import Path\n",
    "from collections import Counter"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.metrics import balanced_accuracy_score\n",
    "from sklearn.metrics import confusion_matrix\n",
    "from imblearn.metrics import classification_report_imbalanced"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Read the CSV and Perform Basic Data Cleaning"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# https://help.lendingclub.com/hc/en-us/articles/215488038-What-do-the-different-Note-statuses-mean-\n",
    "\n",
    "columns = [\n",
    "    \"loan_amnt\", \"int_rate\", \"installment\", \"home_ownership\",\n",
    "    \"annual_inc\", \"verification_status\", \"issue_d\", \"loan_status\",\n",
    "    \"pymnt_plan\", \"dti\", \"delinq_2yrs\", \"inq_last_6mths\",\n",
    "    \"open_acc\", \"pub_rec\", \"revol_bal\", \"total_acc\",\n",
    "    \"initial_list_status\", \"out_prncp\", \"out_prncp_inv\", \"total_pymnt\",\n",
    "    \"total_pymnt_inv\", \"total_rec_prncp\", \"total_rec_int\", \"total_rec_late_fee\",\n",
    "    \"recoveries\", \"collection_recovery_fee\", \"last_pymnt_amnt\", \"next_pymnt_d\",\n",
    "    \"collections_12_mths_ex_med\", \"policy_code\", \"application_type\", \"acc_now_delinq\",\n",
    "    \"tot_coll_amt\", \"tot_cur_bal\", \"open_acc_6m\", \"open_act_il\",\n",
    "    \"open_il_12m\", \"open_il_24m\", \"mths_since_rcnt_il\", \"total_bal_il\",\n",
    "    \"il_util\", \"open_rv_12m\", \"open_rv_24m\", \"max_bal_bc\",\n",
    "    \"all_util\", \"total_rev_hi_lim\", \"inq_fi\", \"total_cu_tl\",\n",
    "    \"inq_last_12m\", \"acc_open_past_24mths\", \"avg_cur_bal\", \"bc_open_to_buy\",\n",
    "    \"bc_util\", \"chargeoff_within_12_mths\", \"delinq_amnt\", \"mo_sin_old_il_acct\",\n",
    "    \"mo_sin_old_rev_tl_op\", \"mo_sin_rcnt_rev_tl_op\", \"mo_sin_rcnt_tl\", \"mort_acc\",\n",
    "    \"mths_since_recent_bc\", \"mths_since_recent_inq\", \"num_accts_ever_120_pd\", \"num_actv_bc_tl\",\n",
    "    \"num_actv_rev_tl\", \"num_bc_sats\", \"num_bc_tl\", \"num_il_tl\",\n",
    "    \"num_op_rev_tl\", \"num_rev_accts\", \"num_rev_tl_bal_gt_0\",\n",
    "    \"num_sats\", \"num_tl_120dpd_2m\", \"num_tl_30dpd\", \"num_tl_90g_dpd_24m\",\n",
    "    \"num_tl_op_past_12m\", \"pct_tl_nvr_dlq\", \"percent_bc_gt_75\", \"pub_rec_bankruptcies\",\n",
    "    \"tax_liens\", \"tot_hi_cred_lim\", \"total_bal_ex_mort\", \"total_bc_limit\",\n",
    "    \"total_il_high_credit_limit\", \"hardship_flag\", \"debt_settlement_flag\"\n",
    "]\n",
    "\n",
    "target = [\"loan_status\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": "   loan_amnt  int_rate  installment home_ownership  annual_inc  \\\n0    10500.0    0.1719       375.35           RENT     66000.0   \n1    25000.0    0.2000       929.09       MORTGAGE    105000.0   \n2    20000.0    0.2000       529.88       MORTGAGE     56000.0   \n3    10000.0    0.1640       353.55           RENT     92000.0   \n4    22000.0    0.1474       520.39       MORTGAGE     52000.0   \n\n  verification_status   issue_d loan_status pymnt_plan    dti  ...  \\\n0     Source Verified  Mar-2019    low_risk          n  27.24  ...   \n1            Verified  Mar-2019    low_risk          n  20.23  ...   \n2            Verified  Mar-2019    low_risk          n  24.26  ...   \n3            Verified  Mar-2019    low_risk          n  31.44  ...   \n4        Not Verified  Mar-2019    low_risk          n  18.76  ...   \n\n   pct_tl_nvr_dlq  percent_bc_gt_75  pub_rec_bankruptcies  tax_liens  \\\n0            85.7             100.0                   0.0        0.0   \n1            91.2              50.0                   1.0        0.0   \n2            66.7              50.0                   0.0        0.0   \n3           100.0              50.0                   1.0        0.0   \n4           100.0               0.0                   0.0        0.0   \n\n   tot_hi_cred_lim  total_bal_ex_mort total_bc_limit  \\\n0          65687.0            38199.0         2000.0   \n1         271427.0            60641.0        41200.0   \n2          60644.0            45684.0         7500.0   \n3          99506.0            68784.0        19700.0   \n4         219750.0            25919.0        27600.0   \n\n   total_il_high_credit_limit  hardship_flag  debt_settlement_flag  \n0                     61987.0              N                     N  \n1                     49197.0              N                     N  \n2                     43144.0              N                     N  \n3                     76506.0              N                     N  \n4                     20000.0              N                     N  \n\n[5 rows x 86 columns]",
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>loan_amnt</th>\n      <th>int_rate</th>\n      <th>installment</th>\n      <th>home_ownership</th>\n      <th>annual_inc</th>\n      <th>verification_status</th>\n      <th>issue_d</th>\n      <th>loan_status</th>\n      <th>pymnt_plan</th>\n      <th>dti</th>\n      <th>...</th>\n      <th>pct_tl_nvr_dlq</th>\n      <th>percent_bc_gt_75</th>\n      <th>pub_rec_bankruptcies</th>\n      <th>tax_liens</th>\n      <th>tot_hi_cred_lim</th>\n      <th>total_bal_ex_mort</th>\n      <th>total_bc_limit</th>\n      <th>total_il_high_credit_limit</th>\n      <th>hardship_flag</th>\n      <th>debt_settlement_flag</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>10500.0</td>\n      <td>0.1719</td>\n      <td>375.35</td>\n      <td>RENT</td>\n      <td>66000.0</td>\n      <td>Source Verified</td>\n      <td>Mar-2019</td>\n      <td>low_risk</td>\n      <td>n</td>\n      <td>27.24</td>\n      <td>...</td>\n      <td>85.7</td>\n      <td>100.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>65687.0</td>\n      <td>38199.0</td>\n      <td>2000.0</td>\n      <td>61987.0</td>\n      <td>N</td>\n      <td>N</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>25000.0</td>\n      <td>0.2000</td>\n      <td>929.09</td>\n      <td>MORTGAGE</td>\n      <td>105000.0</td>\n      <td>Verified</td>\n      <td>Mar-2019</td>\n      <td>low_risk</td>\n      <td>n</td>\n      <td>20.23</td>\n      <td>...</td>\n      <td>91.2</td>\n      <td>50.0</td>\n      <td>1.0</td>\n      <td>0.0</td>\n      <td>271427.0</td>\n      <td>60641.0</td>\n      <td>41200.0</td>\n      <td>49197.0</td>\n      <td>N</td>\n      <td>N</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>20000.0</td>\n      <td>0.2000</td>\n      <td>529.88</td>\n      <td>MORTGAGE</td>\n      <td>56000.0</td>\n      <td>Verified</td>\n      <td>Mar-2019</td>\n      <td>low_risk</td>\n      <td>n</td>\n      <td>24.26</td>\n      <td>...</td>\n      <td>66.7</td>\n      <td>50.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>60644.0</td>\n      <td>45684.0</td>\n      <td>7500.0</td>\n      <td>43144.0</td>\n      <td>N</td>\n      <td>N</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>10000.0</td>\n      <td>0.1640</td>\n      <td>353.55</td>\n      <td>RENT</td>\n      <td>92000.0</td>\n      <td>Verified</td>\n      <td>Mar-2019</td>\n      <td>low_risk</td>\n      <td>n</td>\n      <td>31.44</td>\n      <td>...</td>\n      <td>100.0</td>\n      <td>50.0</td>\n      <td>1.0</td>\n      <td>0.0</td>\n      <td>99506.0</td>\n      <td>68784.0</td>\n      <td>19700.0</td>\n      <td>76506.0</td>\n      <td>N</td>\n      <td>N</td>\n    </tr>\n    <tr>\n      <th>4</th>\n      <td>22000.0</td>\n      <td>0.1474</td>\n      <td>520.39</td>\n      <td>MORTGAGE</td>\n      <td>52000.0</td>\n      <td>Not Verified</td>\n      <td>Mar-2019</td>\n      <td>low_risk</td>\n      <td>n</td>\n      <td>18.76</td>\n      <td>...</td>\n      <td>100.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>219750.0</td>\n      <td>25919.0</td>\n      <td>27600.0</td>\n      <td>20000.0</td>\n      <td>N</td>\n      <td>N</td>\n    </tr>\n  </tbody>\n</table>\n<p>5 rows × 86 columns</p>\n</div>"
     },
     "metadata": {},
     "execution_count": 5
    }
   ],
   "source": [
    "# Load the data\n",
    "file_path = Path('./Resources/LoanStats_2019Q1.csv')\n",
    "df = pd.read_csv(file_path, skiprows=1)[:-2]\n",
    "df = df.loc[:, columns].copy()\n",
    "\n",
    "# Drop the null columns where all values are null\n",
    "df = df.dropna(axis='columns', how='all')\n",
    "\n",
    "# Drop the null rows\n",
    "df = df.dropna()\n",
    "\n",
    "# Remove the `Issued` loan status\n",
    "issued_mask = df['loan_status'] != 'Issued'\n",
    "df = df.loc[issued_mask]\n",
    "\n",
    "# convert interest rate to numerical\n",
    "df['int_rate'] = df['int_rate'].str.replace('%', '')\n",
    "df['int_rate'] = df['int_rate'].astype('float') / 100\n",
    "\n",
    "\n",
    "# Convert the target column values to low_risk and high_risk based on their values\n",
    "x = {'Current': 'low_risk'}   \n",
    "df = df.replace(x)\n",
    "\n",
    "x = dict.fromkeys(['Late (31-120 days)', 'Late (16-30 days)', 'Default', 'In Grace Period'], 'high_risk')    \n",
    "df = df.replace(x)\n",
    "\n",
    "df.reset_index(inplace=True, drop=True)\n",
    "\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Split the Data into Training and Testing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": "0        low_risk\n1        low_risk\n2        low_risk\n3        low_risk\n4        low_risk\n           ...   \n68812    low_risk\n68813    low_risk\n68814    low_risk\n68815    low_risk\n68816    low_risk\nName: loan_status, Length: 68817, dtype: object"
     },
     "metadata": {},
     "execution_count": 6
    }
   ],
   "source": [
    "# Create our features\n",
    "# binary encode using Pandas\n",
    "X= pd.get_dummies(df.drop('loan_status', axis=1))\n",
    "\n",
    "# Create our target\n",
    "y = df[\"loan_status\"]\n",
    "y"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": "          loan_amnt      int_rate   installment    annual_inc           dti  \\\ncount  68817.000000  68817.000000  68817.000000  6.881700e+04  68817.000000   \nmean   16677.594562      0.127718    480.652863  8.821371e+04     21.778153   \nstd    10277.348590      0.048130    288.062432  1.155800e+05     20.199244   \nmin     1000.000000      0.060000     30.890000  4.000000e+01      0.000000   \n25%     9000.000000      0.088100    265.730000  5.000000e+04     13.890000   \n50%    15000.000000      0.118000    404.560000  7.300000e+04     19.760000   \n75%    24000.000000      0.155700    648.100000  1.040000e+05     26.660000   \nmax    40000.000000      0.308400   1676.230000  8.797500e+06    999.000000   \n\n        delinq_2yrs  inq_last_6mths      open_acc       pub_rec  \\\ncount  68817.000000    68817.000000  68817.000000  68817.000000   \nmean       0.217766        0.497697     12.587340      0.126030   \nstd        0.718367        0.758122      6.022869      0.336797   \nmin        0.000000        0.000000      2.000000      0.000000   \n25%        0.000000        0.000000      8.000000      0.000000   \n50%        0.000000        0.000000     11.000000      0.000000   \n75%        0.000000        1.000000     16.000000      0.000000   \nmax       18.000000        5.000000     72.000000      4.000000   \n\n           revol_bal  ...  issue_d_Mar-2019  pymnt_plan_n  \\\ncount   68817.000000  ...      68817.000000       68817.0   \nmean    17604.142828  ...          0.177238           1.0   \nstd     21835.880400  ...          0.381873           0.0   \nmin         0.000000  ...          0.000000           1.0   \n25%      6293.000000  ...          0.000000           1.0   \n50%     12068.000000  ...          0.000000           1.0   \n75%     21735.000000  ...          0.000000           1.0   \nmax    587191.000000  ...          1.000000           1.0   \n\n       initial_list_status_f  initial_list_status_w  next_pymnt_d_Apr-2019  \\\ncount           68817.000000           68817.000000           68817.000000   \nmean                0.123879               0.876121               0.383161   \nstd                 0.329446               0.329446               0.486161   \nmin                 0.000000               0.000000               0.000000   \n25%                 0.000000               1.000000               0.000000   \n50%                 0.000000               1.000000               0.000000   \n75%                 0.000000               1.000000               1.000000   \nmax                 1.000000               1.000000               1.000000   \n\n       next_pymnt_d_May-2019  application_type_Individual  \\\ncount           68817.000000                 68817.000000   \nmean                0.616839                     0.860340   \nstd                 0.486161                     0.346637   \nmin                 0.000000                     0.000000   \n25%                 0.000000                     1.000000   \n50%                 1.000000                     1.000000   \n75%                 1.000000                     1.000000   \nmax                 1.000000                     1.000000   \n\n       application_type_Joint App  hardship_flag_N  debt_settlement_flag_N  \ncount                68817.000000          68817.0                 68817.0  \nmean                     0.139660              1.0                     1.0  \nstd                      0.346637              0.0                     0.0  \nmin                      0.000000              1.0                     1.0  \n25%                      0.000000              1.0                     1.0  \n50%                      0.000000              1.0                     1.0  \n75%                      0.000000              1.0                     1.0  \nmax                      1.000000              1.0                     1.0  \n\n[8 rows x 95 columns]",
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>loan_amnt</th>\n      <th>int_rate</th>\n      <th>installment</th>\n      <th>annual_inc</th>\n      <th>dti</th>\n      <th>delinq_2yrs</th>\n      <th>inq_last_6mths</th>\n      <th>open_acc</th>\n      <th>pub_rec</th>\n      <th>revol_bal</th>\n      <th>...</th>\n      <th>issue_d_Mar-2019</th>\n      <th>pymnt_plan_n</th>\n      <th>initial_list_status_f</th>\n      <th>initial_list_status_w</th>\n      <th>next_pymnt_d_Apr-2019</th>\n      <th>next_pymnt_d_May-2019</th>\n      <th>application_type_Individual</th>\n      <th>application_type_Joint App</th>\n      <th>hardship_flag_N</th>\n      <th>debt_settlement_flag_N</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>count</th>\n      <td>68817.000000</td>\n      <td>68817.000000</td>\n      <td>68817.000000</td>\n      <td>6.881700e+04</td>\n      <td>68817.000000</td>\n      <td>68817.000000</td>\n      <td>68817.000000</td>\n      <td>68817.000000</td>\n      <td>68817.000000</td>\n      <td>68817.000000</td>\n      <td>...</td>\n      <td>68817.000000</td>\n      <td>68817.0</td>\n      <td>68817.000000</td>\n      <td>68817.000000</td>\n      <td>68817.000000</td>\n      <td>68817.000000</td>\n      <td>68817.000000</td>\n      <td>68817.000000</td>\n      <td>68817.0</td>\n      <td>68817.0</td>\n    </tr>\n    <tr>\n      <th>mean</th>\n      <td>16677.594562</td>\n      <td>0.127718</td>\n      <td>480.652863</td>\n      <td>8.821371e+04</td>\n      <td>21.778153</td>\n      <td>0.217766</td>\n      <td>0.497697</td>\n      <td>12.587340</td>\n      <td>0.126030</td>\n      <td>17604.142828</td>\n      <td>...</td>\n      <td>0.177238</td>\n      <td>1.0</td>\n      <td>0.123879</td>\n      <td>0.876121</td>\n      <td>0.383161</td>\n      <td>0.616839</td>\n      <td>0.860340</td>\n      <td>0.139660</td>\n      <td>1.0</td>\n      <td>1.0</td>\n    </tr>\n    <tr>\n      <th>std</th>\n      <td>10277.348590</td>\n      <td>0.048130</td>\n      <td>288.062432</td>\n      <td>1.155800e+05</td>\n      <td>20.199244</td>\n      <td>0.718367</td>\n      <td>0.758122</td>\n      <td>6.022869</td>\n      <td>0.336797</td>\n      <td>21835.880400</td>\n      <td>...</td>\n      <td>0.381873</td>\n      <td>0.0</td>\n      <td>0.329446</td>\n      <td>0.329446</td>\n      <td>0.486161</td>\n      <td>0.486161</td>\n      <td>0.346637</td>\n      <td>0.346637</td>\n      <td>0.0</td>\n      <td>0.0</td>\n    </tr>\n    <tr>\n      <th>min</th>\n      <td>1000.000000</td>\n      <td>0.060000</td>\n      <td>30.890000</td>\n      <td>4.000000e+01</td>\n      <td>0.000000</td>\n      <td>0.000000</td>\n      <td>0.000000</td>\n      <td>2.000000</td>\n      <td>0.000000</td>\n      <td>0.000000</td>\n      <td>...</td>\n      <td>0.000000</td>\n      <td>1.0</td>\n      <td>0.000000</td>\n      <td>0.000000</td>\n      <td>0.000000</td>\n      <td>0.000000</td>\n      <td>0.000000</td>\n      <td>0.000000</td>\n      <td>1.0</td>\n      <td>1.0</td>\n    </tr>\n    <tr>\n      <th>25%</th>\n      <td>9000.000000</td>\n      <td>0.088100</td>\n      <td>265.730000</td>\n      <td>5.000000e+04</td>\n      <td>13.890000</td>\n      <td>0.000000</td>\n      <td>0.000000</td>\n      <td>8.000000</td>\n      <td>0.000000</td>\n      <td>6293.000000</td>\n      <td>...</td>\n      <td>0.000000</td>\n      <td>1.0</td>\n      <td>0.000000</td>\n      <td>1.000000</td>\n      <td>0.000000</td>\n      <td>0.000000</td>\n      <td>1.000000</td>\n      <td>0.000000</td>\n      <td>1.0</td>\n      <td>1.0</td>\n    </tr>\n    <tr>\n      <th>50%</th>\n      <td>15000.000000</td>\n      <td>0.118000</td>\n      <td>404.560000</td>\n      <td>7.300000e+04</td>\n      <td>19.760000</td>\n      <td>0.000000</td>\n      <td>0.000000</td>\n      <td>11.000000</td>\n      <td>0.000000</td>\n      <td>12068.000000</td>\n      <td>...</td>\n      <td>0.000000</td>\n      <td>1.0</td>\n      <td>0.000000</td>\n      <td>1.000000</td>\n      <td>0.000000</td>\n      <td>1.000000</td>\n      <td>1.000000</td>\n      <td>0.000000</td>\n      <td>1.0</td>\n      <td>1.0</td>\n    </tr>\n    <tr>\n      <th>75%</th>\n      <td>24000.000000</td>\n      <td>0.155700</td>\n      <td>648.100000</td>\n      <td>1.040000e+05</td>\n      <td>26.660000</td>\n      <td>0.000000</td>\n      <td>1.000000</td>\n      <td>16.000000</td>\n      <td>0.000000</td>\n      <td>21735.000000</td>\n      <td>...</td>\n      <td>0.000000</td>\n      <td>1.0</td>\n      <td>0.000000</td>\n      <td>1.000000</td>\n      <td>1.000000</td>\n      <td>1.000000</td>\n      <td>1.000000</td>\n      <td>0.000000</td>\n      <td>1.0</td>\n      <td>1.0</td>\n    </tr>\n    <tr>\n      <th>max</th>\n      <td>40000.000000</td>\n      <td>0.308400</td>\n      <td>1676.230000</td>\n      <td>8.797500e+06</td>\n      <td>999.000000</td>\n      <td>18.000000</td>\n      <td>5.000000</td>\n      <td>72.000000</td>\n      <td>4.000000</td>\n      <td>587191.000000</td>\n      <td>...</td>\n      <td>1.000000</td>\n      <td>1.0</td>\n      <td>1.000000</td>\n      <td>1.000000</td>\n      <td>1.000000</td>\n      <td>1.000000</td>\n      <td>1.000000</td>\n      <td>1.000000</td>\n      <td>1.0</td>\n      <td>1.0</td>\n    </tr>\n  </tbody>\n</table>\n<p>8 rows × 95 columns</p>\n</div>"
     },
     "metadata": {},
     "execution_count": 7
    }
   ],
   "source": [
    "X.describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": "low_risk     68470\nhigh_risk      347\nName: loan_status, dtype: int64"
     },
     "metadata": {},
     "execution_count": 8
    }
   ],
   "source": [
    "# Check the balance of our target values\n",
    "df['loan_status'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": "(51612, 95)"
     },
     "metadata": {},
     "execution_count": 9
    }
   ],
   "source": [
    "# Split the X and y into X_train, X_test, y_train, y_test\n",
    "# Use the train_test_split function to create training and testing subsets\n",
    "from sklearn.model_selection import train_test_split\n",
    "X_train, X_test, y_train, y_test = train_test_split(X, \n",
    "                                                    y, \n",
    "                                                    stratify=y)\n",
    "X_train.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Ensemble Learners\n",
    "\n",
    "In this section, you will compare two ensemble algorithms to determine which algorithm results in the best performance. You will train a Balanced Random Forest Classifier and an Easy Ensemble AdaBoost classifier . For each algorithm, be sure to complete the folliowing steps:\n",
    "\n",
    "1. Train the model using the training data. \n",
    "2. Calculate the balanced accuracy score from sklearn.metrics.\n",
    "3. Print the confusion matrix from sklearn.metrics.\n",
    "4. Generate a classication report using the `imbalanced_classification_report` from imbalanced-learn.\n",
    "5. For the Balanced Random Forest Classifier onely, print the feature importance sorted in descending order (most important feature to least important) along with the feature score\n",
    "\n",
    "Note: Use a random state of 1 for each algorithm to ensure consistency between tests"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Balanced Random Forest Classifier"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": "Counter({&#39;low_risk&#39;: 51352, &#39;high_risk&#39;: 260})"
     },
     "metadata": {},
     "execution_count": 10
    }
   ],
   "source": [
    "# Resample the training data with the Balanced Random Forest Classifier\n",
    "from imblearn.ensemble import BalancedRandomForestClassifier\n",
    "\n",
    "brf = BalancedRandomForestClassifier(n_estimators=100)\n",
    "\n",
    "brf.fit(X_train, y_train)\n",
    "Counter(y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": "0.8134218467352373"
     },
     "metadata": {},
     "execution_count": 11
    }
   ],
   "source": [
    "# Calculated the balanced accuracy score\n",
    "y_pred = brf.predict(X_test)\n",
    "balanced_accuracy_score(y_test, y_pred)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": "array([[   65,    22],\n       [ 2059, 15059]], dtype=int64)"
     },
     "metadata": {},
     "execution_count": 12
    }
   ],
   "source": [
    "# Display the confusion matrix\n",
    "confusion_matrix(y_test, y_pred)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": "                   pre       rec       spe        f1       geo       iba       sup\n\n  high_risk       0.03      0.75      0.88      0.06      0.81      0.65        87\n   low_risk       1.00      0.88      0.75      0.94      0.81      0.67     17118\n\navg / total       0.99      0.88      0.75      0.93      0.81      0.67     17205\n\n"
    }
   ],
   "source": [
    "# Print the imbalanced classification report\n",
    "print(classification_report_imbalanced(y_test, y_pred))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": "[(0.07006642197427927, &#39;total_pymnt_inv&#39;),\n (0.06822094336923065, &#39;total_rec_prncp&#39;),\n (0.062484806603050266, &#39;total_pymnt&#39;),\n (0.05428731725936317, &#39;total_rec_int&#39;),\n (0.05159712872297846, &#39;last_pymnt_amnt&#39;),\n (0.02557021600769521, &#39;int_rate&#39;),\n (0.023592443445260017, &#39;issue_d_Jan-2019&#39;),\n (0.020084809544092596, &#39;installment&#39;),\n (0.01950539827292371, &#39;max_bal_bc&#39;),\n (0.018061545940378464, &#39;bc_util&#39;),\n (0.01778950917183341, &#39;mths_since_recent_inq&#39;),\n (0.01677935114666333, &#39;dti&#39;),\n (0.016432096108582053, &#39;out_prncp&#39;),\n (0.01642231198413778, &#39;total_rev_hi_lim&#39;),\n (0.015401381969694827, &#39;mo_sin_old_rev_tl_op&#39;),\n (0.015140038951138518, &#39;mo_sin_old_il_acct&#39;),\n (0.014989794901529995, &#39;annual_inc&#39;),\n (0.01464987441039551, &#39;tot_hi_cred_lim&#39;),\n (0.014606786308550493, &#39;avg_cur_bal&#39;),\n (0.014508824235384363, &#39;total_bal_ex_mort&#39;),\n (0.014203679656393253, &#39;all_util&#39;),\n (0.013953405260396474, &#39;mths_since_rcnt_il&#39;),\n (0.013482301891830872, &#39;total_bc_limit&#39;),\n (0.013384430580255038, &#39;revol_bal&#39;),\n (0.013195968284426329, &#39;bc_open_to_buy&#39;),\n (0.013160803454473582, &#39;num_bc_tl&#39;),\n (0.0125211907382239, &#39;tot_cur_bal&#39;),\n (0.011896247875087751, &#39;il_util&#39;),\n (0.011663516362705177, &#39;out_prncp_inv&#39;),\n (0.01149399041897659, &#39;num_sats&#39;),\n (0.011445523395158065, &#39;issue_d_Mar-2019&#39;),\n (0.011418993904247195, &#39;open_acc&#39;),\n (0.011285519454548589, &#39;num_rev_accts&#39;),\n (0.01120237939851461, &#39;total_il_high_credit_limit&#39;),\n (0.011099282967700248, &#39;total_acc&#39;),\n (0.010992135017105553, &#39;total_bal_il&#39;),\n (0.0108741716016871, &#39;mo_sin_rcnt_rev_tl_op&#39;),\n (0.010782065804512592, &#39;mths_since_recent_bc&#39;),\n (0.010575647384250774, &#39;loan_amnt&#39;),\n (0.009237587385225766, &#39;num_il_tl&#39;),\n (0.008970153562524398, &#39;num_actv_rev_tl&#39;),\n (0.008867107548898955, &#39;mo_sin_rcnt_tl&#39;),\n (0.008490225858731344, &#39;pct_tl_nvr_dlq&#39;),\n (0.00832866664345499, &#39;inq_fi&#39;),\n (0.007897426093402176, &#39;num_actv_bc_tl&#39;),\n (0.007824793246950673, &#39;acc_open_past_24mths&#39;),\n (0.00781413741459785, &#39;num_op_rev_tl&#39;),\n (0.007772760674646359, &#39;total_cu_tl&#39;),\n (0.007574730475502699, &#39;num_rev_tl_bal_gt_0&#39;),\n (0.006518708519756455, &#39;num_tl_op_past_12m&#39;),\n (0.006465681030834307, &#39;num_bc_sats&#39;),\n (0.00638978590824281, &#39;open_rv_24m&#39;),\n (0.006233582277161691, &#39;percent_bc_gt_75&#39;),\n (0.00621096900970759, &#39;inq_last_12m&#39;),\n (0.006194475837055742, &#39;open_il_24m&#39;),\n (0.005754702341503843, &#39;open_acc_6m&#39;),\n (0.0056724505806695445, &#39;mort_acc&#39;),\n (0.00565478726545884, &#39;issue_d_Feb-2019&#39;),\n (0.0055537671893903075, &#39;open_act_il&#39;),\n (0.005394160132654697, &#39;inq_last_6mths&#39;),\n (0.004893827779868524, &#39;total_rec_late_fee&#39;),\n (0.004325281839195452, &#39;open_il_12m&#39;),\n (0.004274193456682694, &#39;next_pymnt_d_Apr-2019&#39;),\n (0.0041293899350397165, &#39;open_rv_12m&#39;),\n (0.003998525448470391, &#39;next_pymnt_d_May-2019&#39;),\n (0.003358616750123203, &#39;delinq_2yrs&#39;),\n (0.0028367538828407857, &#39;tot_coll_amt&#39;),\n (0.0028048065685899416, &#39;verification_status_Verified&#39;),\n (0.0022682061890511008, &#39;home_ownership_OWN&#39;),\n (0.0022467631927478382, &#39;verification_status_Source Verified&#39;),\n (0.0020828823456785135, &#39;home_ownership_RENT&#39;),\n (0.0020031756628580504, &#39;pub_rec&#39;),\n (0.0017699858701502392, &#39;num_accts_ever_120_pd&#39;),\n (0.0015333242805957994, &#39;application_type_Individual&#39;),\n (0.0015209489345618418, &#39;pub_rec_bankruptcies&#39;),\n (0.0015156469358489967, &#39;initial_list_status_w&#39;),\n (0.0014327286858504748, &#39;verification_status_Not Verified&#39;),\n (0.0013810161717307582, &#39;num_tl_90g_dpd_24m&#39;),\n (0.0013742733152588196, &#39;home_ownership_MORTGAGE&#39;),\n (0.0011295861701923471, &#39;initial_list_status_f&#39;),\n (0.0008581418966986071, &#39;application_type_Joint App&#39;),\n (0.00045871544800782203, &#39;collections_12_mths_ex_med&#39;),\n (9.777750406911903e-05, &#39;home_ownership_ANY&#39;),\n (6.251896186211823e-05, &#39;chargeoff_within_12_mths&#39;),\n (0.0, &#39;tax_liens&#39;),\n (0.0, &#39;recoveries&#39;),\n (0.0, &#39;pymnt_plan_n&#39;),\n (0.0, &#39;policy_code&#39;),\n (0.0, &#39;num_tl_30dpd&#39;),\n (0.0, &#39;num_tl_120dpd_2m&#39;),\n (0.0, &#39;hardship_flag_N&#39;),\n (0.0, &#39;delinq_amnt&#39;),\n (0.0, &#39;debt_settlement_flag_N&#39;),\n (0.0, &#39;collection_recovery_fee&#39;),\n (0.0, &#39;acc_now_delinq&#39;)]"
     },
     "metadata": {},
     "execution_count": 14
    }
   ],
   "source": [
    "# List the features sorted in descending order by feature importance\n",
    "# Calculate feature importance in the Balanced Random Forest model.\n",
    "importances = brf.feature_importances_\n",
    "\n",
    "# sort the features by importance\n",
    "sorted(zip(brf.feature_importances_, X.columns), reverse=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Easy Ensemble AdaBoost Classifier"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": "Counter({&#39;low_risk&#39;: 51352, &#39;high_risk&#39;: 260})"
     },
     "metadata": {},
     "execution_count": 15
    }
   ],
   "source": [
    "# Train the Classifier\n",
    "from imblearn.ensemble import EasyEnsembleClassifier\n",
    "eec = EasyEnsembleClassifier(n_estimators=100)\n",
    "eec.fit(X_train, y_train)\n",
    "y_pred = eec.predict(X_test)\n",
    "Counter(y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": "0.9385281071346556"
     },
     "metadata": {},
     "execution_count": 16
    }
   ],
   "source": [
    "# Calculated the balanced accuracy score\n",
    "balanced_accuracy_score(y_test, y_pred)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": "array([[   81,     6],\n       [  924, 16194]], dtype=int64)"
     },
     "metadata": {},
     "execution_count": 17
    }
   ],
   "source": [
    "# Display the confusion matrix\n",
    "confusion_matrix(y_test, y_pred)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": "                   pre       rec       spe        f1       geo       iba       sup\n\n  high_risk       0.08      0.93      0.95      0.15      0.94      0.88        87\n   low_risk       1.00      0.95      0.93      0.97      0.94      0.88     17118\n\navg / total       0.99      0.95      0.93      0.97      0.94      0.88     17205\n\n"
    }
   ],
   "source": [
    "# Print the imbalanced classification report\n",
    "print(classification_report_imbalanced(y_test, y_pred))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "mlenv",
   "language": "python",
   "name": "mlenv"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5-final"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}