Decision_Tree.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Import Library & Load Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn import tree\n",
    "from sklearn.neural_network import MLPClassifier\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "import sklearn\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.metrics import precision_score, recall_score\n",
    "import matplotlib\n",
    "import matplotlib.pyplot as plt\n",
    "from sklearn.neighbors import NearestNeighbors\n",
    "from sklearn.model_selection import GridSearchCV"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.read_csv('./data/ADHD_data_prev.csv')\n",
    "raw = df.to_numpy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(304, 5)"
      ]
     },
     "execution_count": 56,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X = raw[:,2:]\n",
    "X.shape    #(Num of Samples, Num of Features)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(304,)"
      ]
     },
     "execution_count": 57,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y = raw[:,1]\n",
    "y = y.astype('int')\n",
    "y.shape    #(Num of Samples)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Embedding, change to classification"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAh8AAAGdCAYAAACyzRGfAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAAPYQAAD2EBqD+naQAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAblUlEQVR4nO3dcWzedZ3A8c8jlYcN2ioc67NmBYpWBcd2hJm5ybkdsCZzEs2MUYbcOM9E3FCWaQZzf9hLzna36DLN4gxocIhz/iFwXAiwXpRyZlmumywsw3C7MKB30mv0Zlu32UX2vT9wz1Y6lGdrv+XZ83olv4Tn+/u1/eybsr7z2/P0KaSUUgAAZPK2yR4AAKgt4gMAyEp8AABZiQ8AICvxAQBkJT4AgKzEBwCQlfgAALKqm+wBXu/48ePx61//Ourr66NQKEz2OADAm5BSiuHh4Whubo63ve3P39t4y8XHr3/962hpaZnsMQCAM9DX1xczZsz4s9e85eKjvr4+Il4bvqGhYZKnAQDejKGhoWhpaSn/HP9z3nLxceKfWhoaGsQHAFSZN/OUCU84BQCyEh8AQFbiAwDISnwAAFmJDwAgK/EBAGQlPgCArMQHAJCV+AAAshIfAEBW4gMAyEp8AABZiQ8AICvxAQBkVTfZAwBn7op7HpvsESr24volkz0CMMnc+QAAshIfAEBW4gMAyEp8AABZiQ8AICvxAQBk5aW2TAgvAQXgjbjzAQBkJT4AgKwqio+Ojo4oFAqjjlKpVD6fUoqOjo5obm6OKVOmxMKFC2P//v3jPjQAUL0qvvPx/ve/P1555ZXysW/fvvK5DRs2xMaNG2Pz5s3R29sbpVIpFi1aFMPDw+M6NABQvSqOj7q6uiiVSuXj0ksvjYjX7nps2rQp1q1bF0uXLo2ZM2fG1q1b48iRI7Ft27ZxHxwAqE4Vx8eBAweiubk5Wltb49Of/nS88MILERFx8ODB6O/vj/b29vK1xWIxFixYEDt37nzDzzcyMhJDQ0OjDgDg3FVRfMydOzceeOCBePLJJ+O+++6L/v7+mD9/fvz2t7+N/v7+iIhoamoa9TFNTU3lc6fT1dUVjY2N5aOlpeUM/hgAQLWoKD4WL14cn/jEJ+Kaa66Jm266KR577LXf5bB169byNYVCYdTHpJTGrJ1q7dq1MTg4WD76+voqGQkAqDJn9VLbCy+8MK655po4cOBA+VUvr7/LMTAwMOZuyKmKxWI0NDSMOgCAc9dZxcfIyEj86le/iunTp0dra2uUSqXo7u4unz927Fj09PTE/Pnzz3pQAODcUNGvV//KV74SN998c1x22WUxMDAQ//RP/xRDQ0OxfPnyKBQKsWrVqujs7Iy2trZoa2uLzs7OmDp1aixbtmyi5gcAqkxF8fHf//3fccstt8RvfvObuPTSS+ODH/xg7Nq1Ky6//PKIiFizZk0cPXo0VqxYEYcOHYq5c+fGjh07or6+fkKGBwCqT0XxsX379j97vlAoREdHR3R0dJzNTADAOcy72gJZVeM7Hkd412MYT95YDgDISnwAAFmJDwAgK/EBAGQlPgCArLzaBf6kWl+FAVBt3PkAALISHwBAVuIDAMhKfAAAWYkPACAr8QEAZCU+AICsxAcAkJX4AACyEh8AQFbiAwDISnwAAFmJDwAgK/EBAGQlPgCArMQHAJCV+AAAshIfAEBW4gMAyEp8AABZiQ8AICvxAQBkJT4AgKzEBwCQlfgAALISHwBAVuIDAMhKfAAAWYkPACAr8QEAZCU+AICsxAcAkJX4AACyEh8AQFbiAwDISnwAAFmJDwAgK/EBAGQlPgCArMQHAJCV+AAAshIfAEBW4gMAyEp8AABZiQ8AICvxAQBkJT4AgKzEBwCQlfgAALI6q/jo6uqKQqEQq1atKq+llKKjoyOam5tjypQpsXDhwti/f/9ZDwoAnBvOOD56e3vj3nvvjVmzZo1a37BhQ2zcuDE2b94cvb29USqVYtGiRTE8PHzWwwIA1e+M4uP3v/993HrrrXHffffFO9/5zvJ6Sik2bdoU69ati6VLl8bMmTNj69atceTIkdi2bdu4DQ0AVK8zio+VK1fGkiVL4qabbhq1fvDgwejv74/29vbyWrFYjAULFsTOnTvPblIA4JxQV+kHbN++PX75y19Gb2/vmHP9/f0REdHU1DRqvampKV566aXTfr6RkZEYGRkpPx4aGqp0JACgilR056Ovry/uuuuuePDBB+OCCy54w+sKhcKoxymlMWsndHV1RWNjY/loaWmpZCQAoMpUFB979uyJgYGBuO6666Kuri7q6uqip6cnvv3tb0ddXV35jseJOyAnDAwMjLkbcsLatWtjcHCwfPT19Z3hHwUAqAYV/bPLjTfeGPv27Ru19vd///fxvve9L+6+++648soro1QqRXd3d1x77bUREXHs2LHo6emJf/7nfz7t5ywWi1EsFs9wfACg2lQUH/X19TFz5sxRaxdeeGFccskl5fVVq1ZFZ2dntLW1RVtbW3R2dsbUqVNj2bJl4zc1AFC1Kn7C6V+yZs2aOHr0aKxYsSIOHToUc+fOjR07dkR9ff14fykAoAoVUkppsoc41dDQUDQ2Nsbg4GA0NDRM9jicoSvueWyyR4Bx9eL6JZM9ArylVfLz23u7AABZiQ8AICvxAQBkJT4AgKzEBwCQlfgAALISHwBAVuIDAMhKfAAAWYkPACAr8QEAZCU+AICsxAcAkJX4AACyEh8AQFbiAwDISnwAAFmJDwAgK/EBAGQlPgCArMQHAJCV+AAAshIfAEBW4gMAyEp8AABZiQ8AICvxAQBkJT4AgKzEBwCQlfgAALISHwBAVuIDAMhKfAAAWYkPACAr8QEAZCU+AICsxAcAkJX4AACyEh8AQFbiAwDISnwAAFmJDwAgK/EBAGQlPgCArMQHAJCV+AAAshIfAEBW4gMAyEp8AABZiQ8AICvxAQBkJT4AgKzEBwCQlfgAALISHwBAVuIDAMhKfAAAWVUUH1u2bIlZs2ZFQ0NDNDQ0xLx58+Lxxx8vn08pRUdHRzQ3N8eUKVNi4cKFsX///nEfGgCoXhXFx4wZM2L9+vWxe/fu2L17d9xwww3xsY99rBwYGzZsiI0bN8bmzZujt7c3SqVSLFq0KIaHhydkeACg+lQUHzfffHN85CMfife85z3xnve8J77+9a/HRRddFLt27YqUUmzatCnWrVsXS5cujZkzZ8bWrVvjyJEjsW3btomaHwCoMmf8nI9XX301tm/fHocPH4558+bFwYMHo7+/P9rb28vXFIvFWLBgQezcufMNP8/IyEgMDQ2NOgCAc1fF8bFv37646KKLolgsxh133BEPP/xwXH311dHf3x8REU1NTaOub2pqKp87na6urmhsbCwfLS0tlY4EAFSRiuPjve99b+zduzd27doVX/jCF2L58uXx3HPPlc8XCoVR16eUxqydau3atTE4OFg++vr6Kh0JAKgidZV+wPnnnx/vfve7IyJizpw50dvbG9/61rfi7rvvjoiI/v7+mD59evn6gYGBMXdDTlUsFqNYLFY6BgBQpc7693yklGJkZCRaW1ujVCpFd3d3+dyxY8eip6cn5s+ff7ZfBgA4R1R05+OrX/1qLF68OFpaWmJ4eDi2b98eTz31VDzxxBNRKBRi1apV0dnZGW1tbdHW1hadnZ0xderUWLZs2UTNDwBUmYri43//93/jtttui1deeSUaGxtj1qxZ8cQTT8SiRYsiImLNmjVx9OjRWLFiRRw6dCjmzp0bO3bsiPr6+gkZHgCoPoWUUprsIU41NDQUjY2NMTg4GA0NDZM9Dmfoinsem+wRYFy9uH7JZI8Ab2mV/Pz23i4AQFbiAwDISnwAAFmJDwAgK/EBAGQlPgCArMQHAJCV+AAAshIfAEBW4gMAyEp8AABZiQ8AICvxAQBkJT4AgKzEBwCQlfgAALISHwBAVuIDAMhKfAAAWYkPACAr8QEAZCU+AICsxAcAkJX4AACyEh8AQFbiAwDISnwAAFmJDwAgK/EBAGQlPgCArMQHAJCV+AAAshIfAEBW4gMAyKpusgfgL7vinscmewSoedX4/+GL65dM9ghwWu58AABZiQ8AICvxAQBkJT4AgKzEBwCQlfgAALISHwBAVuIDAMhKfAAAWYkPACAr8QEAZCU+AICsxAcAkJX4AACyEh8AQFbiAwDISnwAAFmJDwAgK/EBAGRVN9kDADAxrrjnsckeoWIvrl8y2SOQgTsfAEBW4gMAyKqi+Ojq6ooPfOADUV9fH9OmTYuPf/zj8fzzz4+6JqUUHR0d0dzcHFOmTImFCxfG/v37x3VoAKB6VRQfPT09sXLlyti1a1d0d3fHH//4x2hvb4/Dhw+Xr9mwYUNs3LgxNm/eHL29vVEqlWLRokUxPDw87sMDANWnoiecPvHEE6Me33///TFt2rTYs2dPfPjDH46UUmzatCnWrVsXS5cujYiIrVu3RlNTU2zbti0+//nPj9/kAEBVOqvnfAwODkZExMUXXxwREQcPHoz+/v5ob28vX1MsFmPBggWxc+fO036OkZGRGBoaGnUAAOeuM46PlFKsXr06rr/++pg5c2ZERPT390dERFNT06hrm5qayuder6urKxobG8tHS0vLmY4EAFSBM46PO++8M5599tn48Y9/POZcoVAY9TilNGbthLVr18bg4GD56OvrO9ORAIAqcEa/ZOyLX/xiPProo/H000/HjBkzyuulUikiXrsDMn369PL6wMDAmLshJxSLxSgWi2cyBgBQhSq685FSijvvvDMeeuih+NnPfhatra2jzre2tkapVIru7u7y2rFjx6Knpyfmz58/PhMDAFWtojsfK1eujG3btsW//Mu/RH19ffl5HI2NjTFlypQoFAqxatWq6OzsjLa2tmhra4vOzs6YOnVqLFu2bEL+AABAdakoPrZs2RIREQsXLhy1fv/998ftt98eERFr1qyJo0ePxooVK+LQoUMxd+7c2LFjR9TX14/LwABAdasoPlJKf/GaQqEQHR0d0dHRcaYzAQDnMO/tAgBkJT4AgKzEBwCQlfgAALISHwBAVuIDAMhKfAAAWYkPACAr8QEAZCU+AICsxAcAkJX4AACyEh8AQFYVvastAEykK+55bLJHqNiL65dM9ghVx50PACAr8QEAZCU+AICsxAcAkJX4AACyEh8AQFbiAwDISnwAAFmJDwAgK/EBAGQlPgCArMQHAJCV+AAAshIfAEBW4gMAyEp8AABZiQ8AICvxAQBkJT4AgKzEBwCQlfgAALISHwBAVuIDAMhKfAAAWYkPACAr8QEAZCU+AICsxAcAkJX4AACyEh8AQFbiAwDISnwAAFmJDwAgK/EBAGQlPgCArMQHAJCV+AAAshIfAEBW4gMAyEp8AABZiQ8AICvxAQBkVXF8PP3003HzzTdHc3NzFAqFeOSRR0adTylFR0dHNDc3x5QpU2LhwoWxf//+cRsYAKhuFcfH4cOHY/bs2bF58+bTnt+wYUNs3LgxNm/eHL29vVEqlWLRokUxPDx81sMCANWvrtIPWLx4cSxevPi051JKsWnTpli3bl0sXbo0IiK2bt0aTU1NsW3btvj85z9/dtMCAFVvXJ/zcfDgwejv74/29vbyWrFYjAULFsTOnTvH80sBAFWq4jsff05/f39ERDQ1NY1ab2pqipdeeum0HzMyMhIjIyPlx0NDQ+M5EgDwFjMhr3YpFAqjHqeUxqyd0NXVFY2NjeWjpaVlIkYCAN4ixjU+SqVSRJy8A3LCwMDAmLshJ6xduzYGBwfLR19f33iOBAC8xYxrfLS2tkapVIru7u7y2rFjx6Knpyfmz59/2o8pFovR0NAw6gAAzl0VP+fj97//ffzXf/1X+fHBgwdj7969cfHFF8dll10Wq1atis7Ozmhra4u2trbo7OyMqVOnxrJly8Z1cACgOlUcH7t3746//du/LT9evXp1REQsX748fvCDH8SaNWvi6NGjsWLFijh06FDMnTs3duzYEfX19eM3NQBQtQoppTTZQ5xqaGgoGhsbY3Bw0D/B/MkV9zw22SMA8AZeXL9kskd4S6jk5/e4vtS2GvhBDsB4qsafK5MdTN5YDgDISnwAAFmJDwAgK/EBAGQlPgCArMQHAJCV+AAAshIfAEBW4gMAyEp8AABZiQ8AICvxAQBkJT4AgKzEBwCQlfgAALISHwBAVuIDAMhKfAAAWYkPACAr8QEAZCU+AICsxAcAkJX4AACyEh8AQFbiAwDISnwAAFmJDwAgK/EBAGQlPgCArMQHAJCV+AAAshIfAEBW4gMAyEp8AABZiQ8AICvxAQBkJT4AgKzEBwCQlfgAALISHwBAVuIDAMhKfAAAWYkPACAr8QEAZCU+AICsxAcAkJX4AACyEh8AQFbiAwDISnwAAFmJDwAgK/EBAGQlPgCArMQHAJCV+AAAshIfAEBWExYf3/nOd6K1tTUuuOCCuO666+Lf//3fJ+pLAQBVZELi4yc/+UmsWrUq1q1bF88880z8zd/8TSxevDhefvnlifhyAEAVmZD42LhxY/zDP/xDfO5zn4urrroqNm3aFC0tLbFly5aJ+HIAQBWpG+9PeOzYsdizZ0/cc889o9bb29tj586dY64fGRmJkZGR8uPBwcGIiBgaGhrv0SIi4vjIkQn5vABQLSbiZ+yJz5lS+ovXjnt8/OY3v4lXX301mpqaRq03NTVFf3//mOu7urriH//xH8est7S0jPdoAEBENG6auM89PDwcjY2Nf/aacY+PEwqFwqjHKaUxaxERa9eujdWrV5cfHz9+PP7v//4vLrnkktNen9vQ0FC0tLREX19fNDQ0TPY4k85+nGQvTrIXo9mPk+zFSef6XqSUYnh4OJqbm//iteMeH3/1V38V55133pi7HAMDA2PuhkREFIvFKBaLo9be8Y53jPdYZ62hoeGc/GY5U/bjJHtxkr0YzX6cZC9OOpf34i/d8Thh3J9wev7558d1110X3d3do9a7u7tj/vz54/3lAIAqMyH/7LJ69eq47bbbYs6cOTFv3ry499574+WXX4477rhjIr4cAFBFzuvo6OgY7086c+bMuOSSS6KzszO+8Y1vxNGjR+OHP/xhzJ49e7y/VBbnnXdeLFy4MOrqJuwpMlXFfpxkL06yF6PZj5PsxUn24jWF9GZeEwMAME68twsAkJX4AACyEh8AQFbiAwDISnz8ydNPPx0333xzNDc3R6FQiEceeWTU+ZRSdHR0RHNzc0yZMiUWLlwY+/fvn6RpJ1ZXV1d84AMfiPr6+pg2bVp8/OMfj+eff37UNbWyH1u2bIlZs2aVfynQvHnz4vHHHy+fr5V9OJ2urq4oFAqxatWq8lot7UdHR0cUCoVRR6lUKp+vpb2IiPif//mf+MxnPhOXXHJJTJ06Nf76r/869uzZUz5fS/txxRVXjPneKBQKsXLlyoiorb14I+LjTw4fPhyzZ8+OzZs3n/b8hg0bYuPGjbF58+bo7e2NUqkUixYtiuHh4cyTTryenp5YuXJl7Nq1K7q7u+OPf/xjtLe3x+HDh8vX1Mp+zJgxI9avXx+7d++O3bt3xw033BAf+9jHyn9R1Mo+vF5vb2/ce++9MWvWrFHrtbYf73//++OVV14pH/v27Sufq6W9OHToUHzoQx+Kt7/97fH444/Hc889F9/85jdH/bbqWtqP3t7eUd8XJ37p5ic/+cmIqK29eEOJMSIiPfzww+XHx48fT6VSKa1fv7689oc//CE1Njam7373u5MxYlYDAwMpIlJPT09KyX68853vTN/73vdqdh+Gh4dTW1tb6u7uTgsWLEh33XVXSqn2vi++9rWvpdmzZ5/2XK3txd13352uv/76Nzxfa/vxenfddVd617velY4fP17ze3GCOx9vwsGDB6O/vz/a29vLa8ViMRYsWBA7d+6cxMnyGBwcjIiIiy++OCJqdz9effXV2L59exw+fDjmzZtXs/uwcuXKWLJkSdx0002j1mtxPw4cOBDNzc3R2toan/70p+OFF16IiNrbi0cffTTmzJkTn/zkJ2PatGlx7bXXxn333Vc+X2v7capjx47Fgw8+GJ/97GejUCjU9F6cSny8CSfeJO/1b4zX1NQ05g30zjUppVi9enVcf/31MXPmzIiovf3Yt29fXHTRRVEsFuOOO+6Ihx9+OK6++uqa24eIiO3bt8cvf/nL6OrqGnOu1vZj7ty58cADD8STTz4Z9913X/T398f8+fPjt7/9bc3txQsvvBBbtmyJtra2ePLJJ+OOO+6IL33pS/HAAw9ERO19b5zqkUceid/97ndx++23R0Rt78Wpavv3u1aoUCiMepxSGrN2rrnzzjvj2WefjV/84hdjztXKfrz3ve+NvXv3xu9+97v46U9/GsuXL4+enp7y+VrZh76+vrjrrrtix44dccEFF7zhdbWyH4sXLy7/9zXXXBPz5s2Ld73rXbF169b44Ac/GBG1sxfHjx+POXPmRGdnZ0REXHvttbF///7YsmVL/N3f/V35ulrZj1N9//vfj8WLF495m/la3ItTufPxJpx4Bvvrq3RgYGBMvZ5LvvjFL8ajjz4aP//5z2PGjBnl9Vrbj/PPPz/e/e53x5w5c6Krqytmz54d3/rWt2puH/bs2RMDAwNx3XXXRV1dXdTV1UVPT098+9vfjrq6uvKfuVb24/UuvPDCuOaaa+LAgQM1970xffr0uPrqq0etXXXVVfHyyy9HRO39nXHCSy+9FP/2b/8Wn/vc58prtboXryc+3oTW1tYolUrlZyxHvPbveD09PTF//vxJnGxipJTizjvvjIceeih+9rOfRWtr66jztbYfr5dSipGRkZrbhxtvvDH27dsXe/fuLR9z5syJW2+9Nfbu3RtXXnllTe3H642MjMSvfvWrmD59es19b3zoQx8a83L8//zP/4zLL788Imr374z7778/pk2bFkuWLCmv1epejDFZz3R9qxkeHk7PPPNMeuaZZ1JEpI0bN6ZnnnkmvfTSSymllNavX58aGxvTQw89lPbt25duueWWNH369DQ0NDTJk4+/L3zhC6mxsTE99dRT6ZVXXikfR44cKV9TK/uxdu3a9PTTT6eDBw+mZ599Nn31q19Nb3vb29KOHTtSSrWzD2/k1Fe7pFRb+/HlL385PfXUU+mFF15Iu3btSh/96EdTfX19evHFF1NKtbUX//Ef/5Hq6urS17/+9XTgwIH0ox/9KE2dOjU9+OCD5WtqaT9SSunVV19Nl112Wbr77rvHnKu1vTgd8fEnP//5z1NEjDmWL1+eUnrtpWJf+9rXUqlUSsViMX34wx9O+/btm9yhJ8jp9iEi0v3331++plb247Of/Wy6/PLL0/nnn58uvfTSdOONN5bDI6Xa2Yc38vr4qKX9+NSnPpWmT5+e3v72t6fm5ua0dOnStH///vL5WtqLlFL613/91zRz5sxULBbT+973vnTvvfeOOl9r+/Hkk0+miEjPP//8mHO1thenU0gppUm55QIA1CTP+QAAshIfAEBW4gMAyEp8AABZiQ8AICvxAQBkJT4AgKzEBwCQlfgAALISHwBAVuIDAMhKfAAAWf0/WHcigsqVFiUAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.hist(y)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 1. Threshold - [ y_t ]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "31\n",
      "[69, 61, 61, 64, 64, 65, 73, 65, 64, 68, 72, 68, 62, 71, 66, 61, 75, 69, 69, 61, 66, 65, 72, 64, 65, 66, 61, 65, 70, 63, 64]\n"
     ]
    }
   ],
   "source": [
    "threshold=60\n",
    "over_t = []\n",
    "for i in y:\n",
    "    if i>threshold:\n",
    "        over_t.append(i)\n",
    "print(len(over_t))\n",
    "print(over_t)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0 0 0 0 0 0 0 0 0 1 0 0 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
      " 0 1 0 0 0 0 0 0 0 0 0 0 0 0 0 1 0 0 0 0 0 0 0 1 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
      " 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 0 0 1 0 0 0 0 0 0 0 0 0 0 0 0 1 0 0 0 0\n",
      " 0 0 0 0 0 0 1 0 0 0 0 1 0 0 0 0 0 0 0 1 1 0 0 0 1 0 0 0 0 0 0 0 0 0 0 1 0\n",
      " 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 0 0 1 0 0 0 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
      " 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 0 0 0 0 0 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
      " 0 0 0 0 0 0 0 0 0 1 0 0 0 0 0 0 0 0 0 0 0 0 0 1 0 0 1 0 0 0 0 0 0 0 0 0 0\n",
      " 0 0 1 1 0 0 0 0 0 1 0 0 0 0 1 0 0 1 0 0 0 0 0 0 0 0 0 0 0 0 0 1 1 0 0 0 0\n",
      " 1 0 0 1 0 0 0 0]\n"
     ]
    }
   ],
   "source": [
    "y_t = np.asarray([i>threshold for i in y]).astype('int')\n",
    "print(y_t)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2. Top K% - [ y_k ]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "51\n"
     ]
    }
   ],
   "source": [
    "k = 20\n",
    "cnt = int(k/100*len(y))\n",
    "temp = np.sort(y)[::-1]\n",
    "temp = temp[:cnt]\n",
    "min_val = temp[-1]\n",
    "print(min_val)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0 0 0 0 0 0 0 1 0 1 1 0 1 0 0 0 0 0 0 0 1 0 1 0 0 0 0 0 0 0 0 0 0 1 1 0 1\n",
      " 0 1 0 0 0 0 1 0 0 0 0 0 0 0 0 1 0 0 0 0 0 0 0 1 0 0 0 0 0 1 0 0 0 0 0 0 0\n",
      " 0 1 0 0 0 1 0 0 0 0 0 0 0 0 0 1 1 0 0 1 0 0 0 0 0 0 0 0 0 1 0 0 1 0 0 0 1\n",
      " 0 0 1 0 0 0 1 0 0 0 0 1 0 0 1 0 1 0 0 1 1 0 0 0 1 0 0 0 0 0 0 0 0 0 0 1 0\n",
      " 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 0 0 1 0 0 0 1 0 0 1 1 0 0 0 0 0 0 0 0 1 0 0\n",
      " 0 0 0 0 0 1 0 0 0 0 0 1 0 0 1 0 0 0 0 1 1 0 0 0 0 0 0 0 0 0 1 0 0 1 0 0 1\n",
      " 0 0 0 0 0 0 0 0 0 1 0 0 0 0 0 0 0 0 0 0 0 0 1 1 0 0 1 1 0 0 0 0 0 0 0 0 0\n",
      " 0 0 1 1 1 1 0 0 0 1 0 0 0 0 1 0 0 1 0 0 0 0 0 0 0 0 0 0 0 0 0 1 1 0 0 0 0\n",
      " 1 0 0 1 0 0 0 0]\n"
     ]
    }
   ],
   "source": [
    "y_k = np.asarray([i>=min_val for i in y]).astype('int')\n",
    "print(y_k)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 3. Decide what to use"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {},
   "outputs": [],
   "source": [
    "#y = y_t\n",
    "y = y_k"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Find Hyperparam & Train"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {},
   "outputs": [],
   "source": [
    "params = {\n",
    "    'max_depth': [2, 3, 4, 5, 6, 7],\n",
    "    'min_samples_split': [2, 3, 4, 5, 6, 7, 8, 9, 10]\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "GridSearchCV(cv=3, estimator=RandomForestClassifier(),\n",
       "             param_grid={'max_depth': [2, 3, 4, 5, 6, 7],\n",
       "                         'min_samples_split': [2, 3, 4, 5, 6, 7, 8, 9, 10]})"
      ]
     },
     "execution_count": 66,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "clf = RandomForestClassifier()\n",
    "grid_tree = GridSearchCV(clf, param_grid=params, cv=3, refit=True)\n",
    "grid_tree.fit(X_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "RandomForestClassifier(max_depth=6, min_samples_split=6)\n"
     ]
    }
   ],
   "source": [
    "em = grid_tree.best_estimator_\n",
    "print(em)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# MLP"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "metadata": {},
   "outputs": [],
   "source": [
    "params2 = {\n",
    "    'learning_rate': [\"constant\", \"invscaling\", \"adaptive\"],\n",
    "    'hidden_layer_sizes': [(2,), (3,), (4,), (5, ), (2, 1), (3, 1), (3, 1), (4, 1), (5, 1)],\n",
    "    'activation': [\"logistic\", \"relu\", \"Tanh\"]\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/jameslee/anaconda3/lib/python3.8/site-packages/sklearn/neural_network/_multilayer_perceptron.py:692: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (200) reached and the optimization hasn't converged yet.\n",
      "  warnings.warn(\n",
      "/home/jameslee/anaconda3/lib/python3.8/site-packages/sklearn/neural_network/_multilayer_perceptron.py:692: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (200) reached and the optimization hasn't converged yet.\n",
      "  warnings.warn(\n",
      "/home/jameslee/anaconda3/lib/python3.8/site-packages/sklearn/neural_network/_multilayer_perceptron.py:692: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (200) reached and the optimization hasn't converged yet.\n",
      "  warnings.warn(\n",
      "/home/jameslee/anaconda3/lib/python3.8/site-packages/sklearn/neural_network/_multilayer_perceptron.py:692: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (200) reached and the optimization hasn't converged yet.\n",
      "  warnings.warn(\n",
      "/home/jameslee/anaconda3/lib/python3.8/site-packages/sklearn/neural_network/_multilayer_perceptron.py:692: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (200) reached and the optimization hasn't converged yet.\n",
      "  warnings.warn(\n",
      "/home/jameslee/anaconda3/lib/python3.8/site-packages/sklearn/neural_network/_multilayer_perceptron.py:692: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (200) reached and the optimization hasn't converged yet.\n",
      "  warnings.warn(\n",
      "/home/jameslee/anaconda3/lib/python3.8/site-packages/sklearn/neural_network/_multilayer_perceptron.py:692: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (200) reached and the optimization hasn't converged yet.\n",
      "  warnings.warn(\n",
      "/home/jameslee/anaconda3/lib/python3.8/site-packages/sklearn/neural_network/_multilayer_perceptron.py:692: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (200) reached and the optimization hasn't converged yet.\n",
      "  warnings.warn(\n",
      "/home/jameslee/anaconda3/lib/python3.8/site-packages/sklearn/neural_network/_multilayer_perceptron.py:692: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (200) reached and the optimization hasn't converged yet.\n",
      "  warnings.warn(\n",
      "/home/jameslee/anaconda3/lib/python3.8/site-packages/sklearn/neural_network/_multilayer_perceptron.py:692: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (200) reached and the optimization hasn't converged yet.\n",
      "  warnings.warn(\n",
      "/home/jameslee/anaconda3/lib/python3.8/site-packages/sklearn/neural_network/_multilayer_perceptron.py:692: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (200) reached and the optimization hasn't converged yet.\n",
      "  warnings.warn(\n",
      "/home/jameslee/anaconda3/lib/python3.8/site-packages/sklearn/neural_network/_multilayer_perceptron.py:692: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (200) reached and the optimization hasn't converged yet.\n",
      "  warnings.warn(\n",
      "/home/jameslee/anaconda3/lib/python3.8/site-packages/sklearn/neural_network/_multilayer_perceptron.py:692: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (200) reached and the optimization hasn't converged yet.\n",
      "  warnings.warn(\n",
      "/home/jameslee/anaconda3/lib/python3.8/site-packages/sklearn/neural_network/_multilayer_perceptron.py:692: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (200) reached and the optimization hasn't converged yet.\n",
      "  warnings.warn(\n",
      "/home/jameslee/anaconda3/lib/python3.8/site-packages/sklearn/neural_network/_multilayer_perceptron.py:692: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (200) reached and the optimization hasn't converged yet.\n",
      "  warnings.warn(\n",
      "/home/jameslee/anaconda3/lib/python3.8/site-packages/sklearn/neural_network/_multilayer_perceptron.py:692: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (200) reached and the optimization hasn't converged yet.\n",
      "  warnings.warn(\n",
      "/home/jameslee/anaconda3/lib/python3.8/site-packages/sklearn/neural_network/_multilayer_perceptron.py:692: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (200) reached and the optimization hasn't converged yet.\n",
      "  warnings.warn(\n",
      "/home/jameslee/anaconda3/lib/python3.8/site-packages/sklearn/neural_network/_multilayer_perceptron.py:692: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (200) reached and the optimization hasn't converged yet.\n",
      "  warnings.warn(\n",
      "/home/jameslee/anaconda3/lib/python3.8/site-packages/sklearn/neural_network/_multilayer_perceptron.py:692: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (200) reached and the optimization hasn't converged yet.\n",
      "  warnings.warn(\n",
      "/home/jameslee/anaconda3/lib/python3.8/site-packages/sklearn/neural_network/_multilayer_perceptron.py:692: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (200) reached and the optimization hasn't converged yet.\n",
      "  warnings.warn(\n",
      "/home/jameslee/anaconda3/lib/python3.8/site-packages/sklearn/neural_network/_multilayer_perceptron.py:692: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (200) reached and the optimization hasn't converged yet.\n",
      "  warnings.warn(\n",
      "/home/jameslee/anaconda3/lib/python3.8/site-packages/sklearn/neural_network/_multilayer_perceptron.py:692: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (200) reached and the optimization hasn't converged yet.\n",
      "  warnings.warn(\n",
      "/home/jameslee/anaconda3/lib/python3.8/site-packages/sklearn/neural_network/_multilayer_perceptron.py:692: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (200) reached and the optimization hasn't converged yet.\n",
      "  warnings.warn(\n",
      "/home/jameslee/anaconda3/lib/python3.8/site-packages/sklearn/neural_network/_multilayer_perceptron.py:692: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (200) reached and the optimization hasn't converged yet.\n",
      "  warnings.warn(\n",
      "/home/jameslee/anaconda3/lib/python3.8/site-packages/sklearn/neural_network/_multilayer_perceptron.py:692: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (200) reached and the optimization hasn't converged yet.\n",
      "  warnings.warn(\n",
      "/home/jameslee/anaconda3/lib/python3.8/site-packages/sklearn/neural_network/_multilayer_perceptron.py:692: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (200) reached and the optimization hasn't converged yet.\n",
      "  warnings.warn(\n",
      "/home/jameslee/anaconda3/lib/python3.8/site-packages/sklearn/neural_network/_multilayer_perceptron.py:692: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (200) reached and the optimization hasn't converged yet.\n",
      "  warnings.warn(\n",
      "/home/jameslee/anaconda3/lib/python3.8/site-packages/sklearn/neural_network/_multilayer_perceptron.py:692: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (200) reached and the optimization hasn't converged yet.\n",
      "  warnings.warn(\n",
      "/home/jameslee/anaconda3/lib/python3.8/site-packages/sklearn/neural_network/_multilayer_perceptron.py:692: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (200) reached and the optimization hasn't converged yet.\n",
      "  warnings.warn(\n",
      "/home/jameslee/anaconda3/lib/python3.8/site-packages/sklearn/neural_network/_multilayer_perceptron.py:692: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (200) reached and the optimization hasn't converged yet.\n",
      "  warnings.warn(\n",
      "/home/jameslee/anaconda3/lib/python3.8/site-packages/sklearn/neural_network/_multilayer_perceptron.py:692: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (200) reached and the optimization hasn't converged yet.\n",
      "  warnings.warn(\n",
      "/home/jameslee/anaconda3/lib/python3.8/site-packages/sklearn/neural_network/_multilayer_perceptron.py:692: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (200) reached and the optimization hasn't converged yet.\n",
      "  warnings.warn(\n",
      "/home/jameslee/anaconda3/lib/python3.8/site-packages/sklearn/neural_network/_multilayer_perceptron.py:692: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (200) reached and the optimization hasn't converged yet.\n",
      "  warnings.warn(\n",
      "/home/jameslee/anaconda3/lib/python3.8/site-packages/sklearn/neural_network/_multilayer_perceptron.py:692: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (200) reached and the optimization hasn't converged yet.\n",
      "  warnings.warn(\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/jameslee/anaconda3/lib/python3.8/site-packages/sklearn/neural_network/_multilayer_perceptron.py:692: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (200) reached and the optimization hasn't converged yet.\n",
      "  warnings.warn(\n",
      "/home/jameslee/anaconda3/lib/python3.8/site-packages/sklearn/neural_network/_multilayer_perceptron.py:692: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (200) reached and the optimization hasn't converged yet.\n",
      "  warnings.warn(\n",
      "/home/jameslee/anaconda3/lib/python3.8/site-packages/sklearn/neural_network/_multilayer_perceptron.py:692: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (200) reached and the optimization hasn't converged yet.\n",
      "  warnings.warn(\n",
      "/home/jameslee/anaconda3/lib/python3.8/site-packages/sklearn/neural_network/_multilayer_perceptron.py:692: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (200) reached and the optimization hasn't converged yet.\n",
      "  warnings.warn(\n",
      "/home/jameslee/anaconda3/lib/python3.8/site-packages/sklearn/neural_network/_multilayer_perceptron.py:692: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (200) reached and the optimization hasn't converged yet.\n",
      "  warnings.warn(\n",
      "/home/jameslee/anaconda3/lib/python3.8/site-packages/sklearn/neural_network/_multilayer_perceptron.py:692: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (200) reached and the optimization hasn't converged yet.\n",
      "  warnings.warn(\n",
      "/home/jameslee/anaconda3/lib/python3.8/site-packages/sklearn/neural_network/_multilayer_perceptron.py:692: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (200) reached and the optimization hasn't converged yet.\n",
      "  warnings.warn(\n",
      "/home/jameslee/anaconda3/lib/python3.8/site-packages/sklearn/neural_network/_multilayer_perceptron.py:692: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (200) reached and the optimization hasn't converged yet.\n",
      "  warnings.warn(\n",
      "/home/jameslee/anaconda3/lib/python3.8/site-packages/sklearn/neural_network/_multilayer_perceptron.py:692: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (200) reached and the optimization hasn't converged yet.\n",
      "  warnings.warn(\n",
      "/home/jameslee/anaconda3/lib/python3.8/site-packages/sklearn/neural_network/_multilayer_perceptron.py:692: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (200) reached and the optimization hasn't converged yet.\n",
      "  warnings.warn(\n",
      "/home/jameslee/anaconda3/lib/python3.8/site-packages/sklearn/neural_network/_multilayer_perceptron.py:692: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (200) reached and the optimization hasn't converged yet.\n",
      "  warnings.warn(\n",
      "/home/jameslee/anaconda3/lib/python3.8/site-packages/sklearn/neural_network/_multilayer_perceptron.py:692: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (200) reached and the optimization hasn't converged yet.\n",
      "  warnings.warn(\n",
      "/home/jameslee/anaconda3/lib/python3.8/site-packages/sklearn/neural_network/_multilayer_perceptron.py:692: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (200) reached and the optimization hasn't converged yet.\n",
      "  warnings.warn(\n",
      "/home/jameslee/anaconda3/lib/python3.8/site-packages/sklearn/neural_network/_multilayer_perceptron.py:692: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (200) reached and the optimization hasn't converged yet.\n",
      "  warnings.warn(\n",
      "/home/jameslee/anaconda3/lib/python3.8/site-packages/sklearn/neural_network/_multilayer_perceptron.py:692: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (200) reached and the optimization hasn't converged yet.\n",
      "  warnings.warn(\n",
      "/home/jameslee/anaconda3/lib/python3.8/site-packages/sklearn/neural_network/_multilayer_perceptron.py:692: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (200) reached and the optimization hasn't converged yet.\n",
      "  warnings.warn(\n",
      "/home/jameslee/anaconda3/lib/python3.8/site-packages/sklearn/neural_network/_multilayer_perceptron.py:692: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (200) reached and the optimization hasn't converged yet.\n",
      "  warnings.warn(\n",
      "/home/jameslee/anaconda3/lib/python3.8/site-packages/sklearn/neural_network/_multilayer_perceptron.py:692: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (200) reached and the optimization hasn't converged yet.\n",
      "  warnings.warn(\n",
      "/home/jameslee/anaconda3/lib/python3.8/site-packages/sklearn/neural_network/_multilayer_perceptron.py:692: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (200) reached and the optimization hasn't converged yet.\n",
      "  warnings.warn(\n",
      "/home/jameslee/anaconda3/lib/python3.8/site-packages/sklearn/neural_network/_multilayer_perceptron.py:692: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (200) reached and the optimization hasn't converged yet.\n",
      "  warnings.warn(\n",
      "/home/jameslee/anaconda3/lib/python3.8/site-packages/sklearn/neural_network/_multilayer_perceptron.py:692: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (200) reached and the optimization hasn't converged yet.\n",
      "  warnings.warn(\n",
      "/home/jameslee/anaconda3/lib/python3.8/site-packages/sklearn/neural_network/_multilayer_perceptron.py:692: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (200) reached and the optimization hasn't converged yet.\n",
      "  warnings.warn(\n",
      "/home/jameslee/anaconda3/lib/python3.8/site-packages/sklearn/neural_network/_multilayer_perceptron.py:692: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (200) reached and the optimization hasn't converged yet.\n",
      "  warnings.warn(\n",
      "/home/jameslee/anaconda3/lib/python3.8/site-packages/sklearn/neural_network/_multilayer_perceptron.py:692: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (200) reached and the optimization hasn't converged yet.\n",
      "  warnings.warn(\n",
      "/home/jameslee/anaconda3/lib/python3.8/site-packages/sklearn/neural_network/_multilayer_perceptron.py:692: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (200) reached and the optimization hasn't converged yet.\n",
      "  warnings.warn(\n",
      "/home/jameslee/anaconda3/lib/python3.8/site-packages/sklearn/neural_network/_multilayer_perceptron.py:692: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (200) reached and the optimization hasn't converged yet.\n",
      "  warnings.warn(\n",
      "/home/jameslee/anaconda3/lib/python3.8/site-packages/sklearn/neural_network/_multilayer_perceptron.py:692: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (200) reached and the optimization hasn't converged yet.\n",
      "  warnings.warn(\n",
      "/home/jameslee/anaconda3/lib/python3.8/site-packages/sklearn/neural_network/_multilayer_perceptron.py:692: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (200) reached and the optimization hasn't converged yet.\n",
      "  warnings.warn(\n",
      "/home/jameslee/anaconda3/lib/python3.8/site-packages/sklearn/neural_network/_multilayer_perceptron.py:692: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (200) reached and the optimization hasn't converged yet.\n",
      "  warnings.warn(\n",
      "/home/jameslee/anaconda3/lib/python3.8/site-packages/sklearn/neural_network/_multilayer_perceptron.py:692: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (200) reached and the optimization hasn't converged yet.\n",
      "  warnings.warn(\n",
      "/home/jameslee/anaconda3/lib/python3.8/site-packages/sklearn/neural_network/_multilayer_perceptron.py:692: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (200) reached and the optimization hasn't converged yet.\n",
      "  warnings.warn(\n",
      "/home/jameslee/anaconda3/lib/python3.8/site-packages/sklearn/neural_network/_multilayer_perceptron.py:692: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (200) reached and the optimization hasn't converged yet.\n",
      "  warnings.warn(\n",
      "/home/jameslee/anaconda3/lib/python3.8/site-packages/sklearn/neural_network/_multilayer_perceptron.py:692: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (200) reached and the optimization hasn't converged yet.\n",
      "  warnings.warn(\n",
      "/home/jameslee/anaconda3/lib/python3.8/site-packages/sklearn/neural_network/_multilayer_perceptron.py:692: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (200) reached and the optimization hasn't converged yet.\n",
      "  warnings.warn(\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/jameslee/anaconda3/lib/python3.8/site-packages/sklearn/neural_network/_multilayer_perceptron.py:692: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (200) reached and the optimization hasn't converged yet.\n",
      "  warnings.warn(\n",
      "/home/jameslee/anaconda3/lib/python3.8/site-packages/sklearn/neural_network/_multilayer_perceptron.py:692: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (200) reached and the optimization hasn't converged yet.\n",
      "  warnings.warn(\n",
      "/home/jameslee/anaconda3/lib/python3.8/site-packages/sklearn/neural_network/_multilayer_perceptron.py:692: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (200) reached and the optimization hasn't converged yet.\n",
      "  warnings.warn(\n",
      "/home/jameslee/anaconda3/lib/python3.8/site-packages/sklearn/neural_network/_multilayer_perceptron.py:692: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (200) reached and the optimization hasn't converged yet.\n",
      "  warnings.warn(\n",
      "/home/jameslee/anaconda3/lib/python3.8/site-packages/sklearn/neural_network/_multilayer_perceptron.py:692: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (200) reached and the optimization hasn't converged yet.\n",
      "  warnings.warn(\n",
      "/home/jameslee/anaconda3/lib/python3.8/site-packages/sklearn/neural_network/_multilayer_perceptron.py:692: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (200) reached and the optimization hasn't converged yet.\n",
      "  warnings.warn(\n",
      "/home/jameslee/anaconda3/lib/python3.8/site-packages/sklearn/neural_network/_multilayer_perceptron.py:692: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (200) reached and the optimization hasn't converged yet.\n",
      "  warnings.warn(\n",
      "/home/jameslee/anaconda3/lib/python3.8/site-packages/sklearn/neural_network/_multilayer_perceptron.py:692: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (200) reached and the optimization hasn't converged yet.\n",
      "  warnings.warn(\n",
      "/home/jameslee/anaconda3/lib/python3.8/site-packages/sklearn/neural_network/_multilayer_perceptron.py:692: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (200) reached and the optimization hasn't converged yet.\n",
      "  warnings.warn(\n",
      "/home/jameslee/anaconda3/lib/python3.8/site-packages/sklearn/neural_network/_multilayer_perceptron.py:692: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (200) reached and the optimization hasn't converged yet.\n",
      "  warnings.warn(\n",
      "/home/jameslee/anaconda3/lib/python3.8/site-packages/sklearn/neural_network/_multilayer_perceptron.py:692: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (200) reached and the optimization hasn't converged yet.\n",
      "  warnings.warn(\n",
      "/home/jameslee/anaconda3/lib/python3.8/site-packages/sklearn/neural_network/_multilayer_perceptron.py:692: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (200) reached and the optimization hasn't converged yet.\n",
      "  warnings.warn(\n",
      "/home/jameslee/anaconda3/lib/python3.8/site-packages/sklearn/neural_network/_multilayer_perceptron.py:692: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (200) reached and the optimization hasn't converged yet.\n",
      "  warnings.warn(\n",
      "/home/jameslee/anaconda3/lib/python3.8/site-packages/sklearn/neural_network/_multilayer_perceptron.py:692: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (200) reached and the optimization hasn't converged yet.\n",
      "  warnings.warn(\n",
      "/home/jameslee/anaconda3/lib/python3.8/site-packages/sklearn/neural_network/_multilayer_perceptron.py:692: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (200) reached and the optimization hasn't converged yet.\n",
      "  warnings.warn(\n",
      "/home/jameslee/anaconda3/lib/python3.8/site-packages/sklearn/neural_network/_multilayer_perceptron.py:692: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (200) reached and the optimization hasn't converged yet.\n",
      "  warnings.warn(\n",
      "/home/jameslee/anaconda3/lib/python3.8/site-packages/sklearn/neural_network/_multilayer_perceptron.py:692: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (200) reached and the optimization hasn't converged yet.\n",
      "  warnings.warn(\n",
      "/home/jameslee/anaconda3/lib/python3.8/site-packages/sklearn/neural_network/_multilayer_perceptron.py:692: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (200) reached and the optimization hasn't converged yet.\n",
      "  warnings.warn(\n",
      "/home/jameslee/anaconda3/lib/python3.8/site-packages/sklearn/neural_network/_multilayer_perceptron.py:692: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (200) reached and the optimization hasn't converged yet.\n",
      "  warnings.warn(\n",
      "/home/jameslee/anaconda3/lib/python3.8/site-packages/sklearn/neural_network/_multilayer_perceptron.py:692: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (200) reached and the optimization hasn't converged yet.\n",
      "  warnings.warn(\n",
      "/home/jameslee/anaconda3/lib/python3.8/site-packages/sklearn/neural_network/_multilayer_perceptron.py:692: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (200) reached and the optimization hasn't converged yet.\n",
      "  warnings.warn(\n",
      "/home/jameslee/anaconda3/lib/python3.8/site-packages/sklearn/neural_network/_multilayer_perceptron.py:692: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (200) reached and the optimization hasn't converged yet.\n",
      "  warnings.warn(\n",
      "/home/jameslee/anaconda3/lib/python3.8/site-packages/sklearn/neural_network/_multilayer_perceptron.py:692: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (200) reached and the optimization hasn't converged yet.\n",
      "  warnings.warn(\n",
      "/home/jameslee/anaconda3/lib/python3.8/site-packages/sklearn/neural_network/_multilayer_perceptron.py:692: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (200) reached and the optimization hasn't converged yet.\n",
      "  warnings.warn(\n",
      "/home/jameslee/anaconda3/lib/python3.8/site-packages/sklearn/neural_network/_multilayer_perceptron.py:692: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (200) reached and the optimization hasn't converged yet.\n",
      "  warnings.warn(\n",
      "/home/jameslee/anaconda3/lib/python3.8/site-packages/sklearn/model_selection/_validation.py:372: FitFailedWarning: \n",
      "81 fits failed out of a total of 243.\n",
      "The score on these train-test partitions for these parameters will be set to nan.\n",
      "If these failures are not expected, you can try to debug them by setting error_score='raise'.\n",
      "\n",
      "Below are more details about the failures:\n",
      "--------------------------------------------------------------------------------\n",
      "81 fits failed with the following error:\n",
      "Traceback (most recent call last):\n",
      "  File \"/home/jameslee/anaconda3/lib/python3.8/site-packages/sklearn/model_selection/_validation.py\", line 680, in _fit_and_score\n",
      "    estimator.fit(X_train, y_train, **fit_params)\n",
      "  File \"/home/jameslee/anaconda3/lib/python3.8/site-packages/sklearn/neural_network/_multilayer_perceptron.py\", line 752, in fit\n",
      "    return self._fit(X, y, incremental=False)\n",
      "  File \"/home/jameslee/anaconda3/lib/python3.8/site-packages/sklearn/neural_network/_multilayer_perceptron.py\", line 384, in _fit\n",
      "    self._validate_hyperparameters()\n",
      "  File \"/home/jameslee/anaconda3/lib/python3.8/site-packages/sklearn/neural_network/_multilayer_perceptron.py\", line 493, in _validate_hyperparameters\n",
      "    raise ValueError(\n",
      "ValueError: The activation 'Tanh' is not supported. Supported activations are ['identity', 'logistic', 'relu', 'softmax', 'tanh'].\n",
      "\n",
      "  warnings.warn(some_fits_failed_message, FitFailedWarning)\n",
      "/home/jameslee/anaconda3/lib/python3.8/site-packages/sklearn/model_selection/_search.py:969: UserWarning: One or more of the test scores are non-finite: [0.59259259 0.39917695 0.60082305 0.59259259 0.60082305 0.40740741\n",
      " 0.79423868 0.79012346 0.60082305 0.79423868 0.59259259 0.79423868\n",
      " 0.79423868 0.39917695 0.79423868 0.59259259 0.60082305 0.79423868\n",
      " 0.79423868 0.79423868 0.60082305 0.60082305 0.79423868 0.60082305\n",
      " 0.40740741 0.79423868 0.39917695 0.39917695 0.79423868 0.60082305\n",
      " 0.60082305 0.79423868 0.59259259 0.79423868 0.67901235 0.40740741\n",
      " 0.781893   0.79423868 0.36213992 0.40329218 0.79423868 0.60082305\n",
      " 0.60082305 0.39917695 0.20164609 0.39917695 0.39917695 0.49382716\n",
      " 0.60082305 0.40740741 0.39506173 0.56378601 0.59259259 0.60082305\n",
      "        nan        nan        nan        nan        nan        nan\n",
      "        nan        nan        nan        nan        nan        nan\n",
      "        nan        nan        nan        nan        nan        nan\n",
      "        nan        nan        nan        nan        nan        nan\n",
      "        nan        nan        nan]\n",
      "  warnings.warn(\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "GridSearchCV(cv=3, estimator=MLPClassifier(),\n",
       "             param_grid={'activation': ['logistic', 'relu', 'Tanh'],\n",
       "                         'hidden_layer_sizes': [(2,), (3,), (4,), (5,), (2, 1),\n",
       "                                                (3, 1), (3, 1), (4, 1),\n",
       "                                                (5, 1)],\n",
       "                         'learning_rate': ['constant', 'invscaling',\n",
       "                                           'adaptive']})"
      ]
     },
     "execution_count": 78,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "clf2 = MLPClassifier()\n",
    "grid_mlp = GridSearchCV(clf2, param_grid=params2, cv=3, refit=True)\n",
    "grid_mlp.fit(X_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "MLPClassifier(activation='logistic', hidden_layer_sizes=(4,))\n"
     ]
    }
   ],
   "source": [
    "em2 = grid_mlp.best_estimator_\n",
    "print(em2)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Prediction & Metrics"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.819672131147541\n"
     ]
    }
   ],
   "source": [
    "prediction = em.predict(X_test)\n",
    "confidence = em.predict_proba(X_test)\n",
    "score = em.score(X_test, y_test)\n",
    "print(\"Accuracy: \"+str(score))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.819672131147541\n"
     ]
    }
   ],
   "source": [
    "prediction2 = em2.predict(X_test)\n",
    "confidence2 = em2.predict_proba(X_test)\n",
    "score2 = em2.score(X_test, y_test)\n",
    "print(\"Accuracy: \"+str(score2))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Precision: 0.0\n",
      "Recall: 0.0\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/jameslee/anaconda3/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1318: UndefinedMetricWarning: Precision is ill-defined and being set to 0.0 due to no predicted samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n"
     ]
    }
   ],
   "source": [
    "precision = precision_score(y_test, prediction)\n",
    "recall = recall_score(y_test, prediction)\n",
    "print(\"Precision: \"+str(precision))\n",
    "print(\"Recall: \"+str(recall))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Precision: 0.0\n",
      "Recall: 0.0\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/jameslee/anaconda3/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1318: UndefinedMetricWarning: Precision is ill-defined and being set to 0.0 due to no predicted samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n"
     ]
    }
   ],
   "source": [
    "precision = precision_score(y_test, prediction2)\n",
    "recall = recall_score(y_test, prediction2)\n",
    "print(\"Precision: \"+str(precision))\n",
    "print(\"Recall: \"+str(recall))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0 1 0 1 0 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 1 0 0 1 0 1 0 0 0 0 0 0 0 0 0\n",
      " 1 0 0 0 0 0 0 1 0 0 0 0 1 0 0 0 0 1 0 0 0 0 0 0]\n"
     ]
    }
   ],
   "source": [
    "print(y_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
      " 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0]\n"
     ]
    }
   ],
   "source": [
    "print(prediction)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
      " 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0]\n"
     ]
    }
   ],
   "source": [
    "print(prediction2)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}