Model_1.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Personalized Medicine: Redefining Cancer Treatment\n",
    "\n",
    "\n",
    "\n",
    "#### Matt Shaffer https://github.com/planetceres \n",
    "#### Kaggle Competition: https://www.kaggle.com/c/msk-redefining-cancer-treatment"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "According to [discussion boards](https://www.kaggle.com/c/msk-redefining-cancer-treatment/discussion/35810#202604) on Kaggle, the classes we are trying to predict appear to be as follows:\n",
    "\n",
    "1. Likely Loss-of-function\n",
    "2. Likely Gain-of-function\n",
    "3. Neutral\n",
    "4. Loss-of-function\n",
    "5. Likely Neutral\n",
    "6. Inconclusive\n",
    "7. Gain-of-function\n",
    "8. Likely Switch-of-function\n",
    "9. Switch-of-function\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Dependencies"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import os\n",
    "import time\n",
    "import glob\n",
    "import re\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import scipy.sparse as sps\n",
    "import Bio\n",
    "\n",
    "from sklearn.decomposition import TruncatedSVD\n",
    "from sklearn.preprocessing import LabelEncoder\n",
    "from sklearn.feature_extraction.text import CountVectorizer\n",
    "from sklearn.feature_extraction.text import TfidfTransformer\n",
    "from sklearn.feature_extraction.text import TfidfVectorizer\n",
    "from sklearn.model_selection import GridSearchCV\n",
    "from sklearn.metrics import make_scorer\n",
    "from sklearn.metrics import explained_variance_score\n",
    "from sklearn.pipeline import make_pipeline\n",
    "\n",
    "from nltk.stem import PorterStemmer, WordNetLemmatizer\n",
    "from nltk.tokenize import word_tokenize\n",
    "from nltk.corpus import stopwords\n",
    "\n",
    "from keras.models import Sequential\n",
    "from keras.layers import Dense\n",
    "from keras.wrappers.scikit_learn import KerasClassifier\n",
    "from keras.utils import np_utils\n",
    "from keras.layers import Dropout\n",
    "from keras.utils import np_utils\n",
    "from keras.models import model_from_json\n",
    "from keras.callbacks import ModelCheckpoint, EarlyStopping\n",
    "\n",
    "%matplotlib inline\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "import logging\n",
    "logging.basicConfig(level=logging.INFO, format='%(asctime)s %(message)s')\n",
    "from itertools import islice"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "model_version = '001'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "model_name = 'model_' + model_version"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "data_directory = '/Data/kaggle/Personalized Medicine'\n",
    "model_directory = data_directory + '/saved_models'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "model_path = os.path.join(model_directory, model_name)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Create model directory if it does not exist\n",
    "if not os.path.isdir(model_directory):\n",
    "    print(\"creating directory for saved models\")\n",
    "    os.mkdir(model_directory)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Load model to resume training or perform inference\n",
    "def load_model_from_json(model_path):\n",
    "    model = model_from_json(open(model_path + '.json').read())\n",
    "    model.load_weights(model_path + '.h5')\n",
    "    #model.compile(optimizer=rmsprop, loss='mse')\n",
    "    return model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from keras.models import load_model\n",
    "# Load model to resume training or perform inference\n",
    "def load_recent_model(model_path):\n",
    "    # Locate the most recent model the folder to resume training from\n",
    "    model_recent = max(glob.iglob(model_path + '*.hdf5'), key=os.path.getctime)\n",
    "    print(\"Using model at checkpoint: {}\".format(model_recent))\n",
    "    #model = model_from_json(open(model_path + '.json').read())\n",
    "    model = load_model(model_recent)\n",
    "    #model.compile(optimizer=rmsprop, loss='mse')\n",
    "    return model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Save model\n",
    "def save_model_to_json(m, model_path):    \n",
    "    json_string = m.model.to_json()\n",
    "    open(model_path + '.json', 'w').write(json_string)\n",
    "    m.model.save_weights(model_path + '.h5', overwrite=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def print_op_str(data_type):\n",
    "    p = \"Done processing \" + data_type + \" data in {:.2f} seconds\"\n",
    "    return p"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def print_blank(n):\n",
    "    print(\" \"*n, end=\"\\r\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Data Overview"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "train_variants = pd.read_csv(data_directory + \"/input/training_variants\")\n",
    "test_variants = pd.read_csv(data_directory + \"/input/test_variants\")\n",
    "train_text = pd.read_csv(data_directory + \"/input/training_text\", sep=\"\\|\\|\", engine='python', header=None, skiprows=1, names=[\"ID\",\"Text\"])\n",
    "test_text = pd.read_csv(data_directory + \"/input/test_text\", sep=\"\\|\\|\", engine='python', header=None, skiprows=1, names=[\"ID\",\"Text\"])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The test set has no labels and is used only for score submission. This will be a challenge since the sample size is small, and it will be hard to learn the properties of the population needed to perform inference. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['ID', 'Gene', 'Variation', 'Class']\n",
      "['ID', 'Gene', 'Variation']\n"
     ]
    }
   ],
   "source": [
    "# Test set has no labels and is used \n",
    "print(list(train_variants.columns))\n",
    "print(list(test_variants.columns))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "In addition to the gene variant data, we also have a text corpus for each example that provides the clinical evidence that human experts used to classify the genetic mutations. This is essentially an unstructured feature set, and our first task will be to map this noisy data to a set of features that can more easily be used for prediction. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['ID', 'Text']\n",
      "['ID', 'Text']\n"
     ]
    }
   ],
   "source": [
    "print(list(train_text.columns))\n",
    "print(list(test_text.columns))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Merge the text with the variant data, and separate the target values (`Class`) from the features\n",
    "train = pd.merge(train_variants, train_text, how='left', on='ID')\n",
    "y_train = train['Class'].values\n",
    "X_train = train.drop('Class', axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Do the same thing with the test data, but note that there are no classes to separate as targets\n",
    "X_test = pd.merge(test_variants, test_text, how='left', on='ID')\n",
    "test_index = X_test['ID'].values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Create mini data sets for model building\n",
    "train_mini = train.sample(frac=0.05)\n",
    "y_train_mini = train_mini['Class'].values\n",
    "X_train_mini = train_mini.drop('Class', axis=1)\n",
    "X_test_mini = X_test.sample(frac=0.05)\n",
    "test_index_mini = X_test_mini['ID'].values\n",
    "\n",
    "# Create mini dev set for model building\n",
    "dev_mini = train.sample(frac=0.05)\n",
    "y_dev_mini = dev_mini['Class'].values\n",
    "X_dev_mini = dev_mini.drop('Class', axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(166, 4)"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_train_mini.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# create dataset with all variants\n",
    "all_variants = pd.concat([train_variants, test_variants], ignore_index=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Save/Load tokenized vocabulary"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def save_sparse_csr(filename,array):\n",
    "    np.savez(filename,data = array.data ,indices=array.indices,\n",
    "             indptr =array.indptr, shape=array.shape )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def load_sparse_csr(filename):\n",
    "    loader = np.load(filename + '.npz')\n",
    "    return sps.csr_matrix((  loader['data'], loader['indices'], loader['indptr']),\n",
    "                         shape = loader['shape'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 406 ms, sys: 599 ms, total: 1.01 s\n",
      "Wall time: 1.23 s\n"
     ]
    }
   ],
   "source": [
    "save_sparse_csr(data_directory + '/data/train_bigram_vocabulary', tfidf_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 587 ms, sys: 797 ms, total: 1.38 s\n",
      "Wall time: 1.63 s\n"
     ]
    }
   ],
   "source": [
    "save_sparse_csr(data_directory + '/data/test_bigram_vocabulary', tfidf_test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Load tfidf from previous session if applicable:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 911 ms, sys: 345 ms, total: 1.26 s\n",
      "Wall time: 1.31 s\n"
     ]
    }
   ],
   "source": [
    "%%time \n",
    "tfidf = TfidfVectorizer()\n",
    "tfidf_train = load_sparse_csr(data_directory + '/data/train_bigram_vocabulary')\n",
    "tfidf_test = load_sparse_csr(data_directory + '/data/test_bigram_vocabulary')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "In order to reduce the dimensionality of the transformed text, we can use truncated singular value decomposition (SVD), which is similar to PCA, but for sparse matrices. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Choose index of number of features where we will truncate dimensionality\n",
    "features_n_svd = 200"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Load previously transformed data if applicable."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "svd_train = np.load(data_directory + '/data/train_svd_200.npy')\n",
    "svd_test = np.load(data_directory + '/data/test_svd_200.npy')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "%%time\n",
    "svd = TruncatedSVD(features_n_svd, algorithm='arpack')\n",
    "svd_train = svd.fit_transform(tfidf_train)\n",
    "svd_test = svd.transform(tfidf_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(3321, 200)"
      ]
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "svd_train.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "np.save(data_directory + '/data/train_svd_200', svd_train)\n",
    "np.save(data_directory + '/data/test_svd_200', svd_test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Fully Connected Neural Network"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**  Hyperparameters **"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "input_shape = svd_train.shape[1]\n",
    "output_shape = len(train['Class'].unique())\n",
    "batch_n = 32\n",
    "EPOCHS_N = 100\n",
    "model_save_interval = 100"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Model"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "This still seems to be the best model, although training time limited the amount of models that were tested. \n",
    "hidden units => `512` => `256` => `128` => `64`), then (`64` => `128` => `256` => `512`)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def model_hypothesis():\n",
    "    model = Sequential()\n",
    "    model.add(Dense(512, input_dim=input_shape, kernel_initializer='normal', activation='relu'))\n",
    "    model.add(Dropout(0.5))\n",
    "    model.add(Dense(256, kernel_initializer='normal', activation='relu'))\n",
    "    model.add(Dropout(0.5))\n",
    "    model.add(Dense(128, kernel_initializer='normal', activation='relu'))\n",
    "    model.add(Dropout(0.5))\n",
    "    model.add(Dense(64, kernel_initializer='normal', activation='relu'))\n",
    "    model.add(Dropout(0.5))\n",
    "    model.add(Dense(128, kernel_initializer='normal', activation='relu'))\n",
    "    model.add(Dropout(0.5))\n",
    "    model.add(Dense(256, kernel_initializer='normal', activation='relu'))\n",
    "    model.add(Dropout(0.5))\n",
    "    model.add(Dense(512, kernel_initializer='normal', activation='relu'))\n",
    "    model.add(Dropout(0.5))\n",
    "    model.add(Dense(output_shape, kernel_initializer='normal', activation=\"softmax\"))\n",
    "    model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'])\n",
    "    return model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def model_resume():\n",
    "    model = load_recent_model(model_path)\n",
    "    model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'])\n",
    "    return model"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "One-hot encoding of labels"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Callback for saving model and weights at n intervals in training\n",
    "# https://keras.io/callbacks/#modelcheckpoint\n",
    "weight_save_callback = ModelCheckpoint(model_path + '.{epoch:02d}-{loss:.2f}.hdf5', \n",
    "                                       monitor='loss', \n",
    "                                       verbose=0, \n",
    "                                       save_best_only=True, \n",
    "                                       mode='auto',\n",
    "                                       period=model_save_interval # Interval (number of epochs) between checkpoints\n",
    "                                      )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "early_stopping = EarlyStopping(monitor='loss', min_delta=0, patience=50, verbose=1, mode='auto')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "onehot = LabelEncoder()\n",
    "onehot.fit(y_train)\n",
    "y_enc = onehot.transform(y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "y_ind = np_utils.to_categorical(y_enc)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "To restore a previously saved model:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Using model at checkpoint: /Data/kaggle/Personalized Medicine/saved_models/model_001.244-0.73.hdf5\n"
     ]
    }
   ],
   "source": [
    "# Try to restore previous checkpoints to continue training\n",
    "if os.path.isfile(model_path + '.h5') and os.path.isfile(model_path + '.json'):\n",
    "    estimator = KerasClassifier(build_fn=model_resume, epochs=EPOCHS_N, batch_size=batch_n)\n",
    "    model = model_resume()\n",
    "else:\n",
    "    estimator = KerasClassifier(build_fn=model_hypothesis, epochs=EPOCHS_N, batch_size=batch_n)\n",
    "    model = model_hypothesis()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Or create a new one:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "estimator = KerasClassifier(build_fn=model_hypothesis, epochs=EPOCHS_N, batch_size=batch_n)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Using model at checkpoint: /Data/kaggle/Personalized Medicine/saved_models/model_001.244-0.73.hdf5\n",
      "Epoch 1/1000\n",
      "3321/3321 [==============================] - 2s - loss: 0.7426 - acc: 0.7368     \n",
      "Epoch 2/1000\n",
      "3321/3321 [==============================] - 0s - loss: 0.7757 - acc: 0.7392     \n",
      "Epoch 3/1000\n",
      "3321/3321 [==============================] - 0s - loss: 0.7661 - acc: 0.7347     \n",
      "Epoch 4/1000\n",
      "3321/3321 [==============================] - 1s - loss: 0.7691 - acc: 0.7383     \n",
      "Epoch 5/1000\n",
      "3321/3321 [==============================] - 1s - loss: 0.7486 - acc: 0.7422     \n",
      "Epoch 6/1000\n",
      "3321/3321 [==============================] - 0s - loss: 0.7433 - acc: 0.7404     \n",
      "Epoch 7/1000\n",
      "3321/3321 [==============================] - 1s - loss: 0.7888 - acc: 0.7332     \n",
      "Epoch 8/1000\n",
      "3321/3321 [==============================] - 0s - loss: 0.7780 - acc: 0.7356     \n",
      "Epoch 9/1000\n",
      "3321/3321 [==============================] - 1s - loss: 0.7770 - acc: 0.7444     \n",
      "Epoch 10/1000\n",
      "3321/3321 [==============================] - 1s - loss: 0.7798 - acc: 0.7305     \n",
      "Epoch 11/1000\n",
      "3321/3321 [==============================] - 1s - loss: 0.7667 - acc: 0.7338     \n",
      "Epoch 12/1000\n",
      "3321/3321 [==============================] - 1s - loss: 0.7518 - acc: 0.7392     \n",
      "Epoch 13/1000\n",
      "3321/3321 [==============================] - 1s - loss: 0.7459 - acc: 0.7350     \n",
      "Epoch 14/1000\n",
      "3321/3321 [==============================] - 1s - loss: 0.7592 - acc: 0.7356     \n",
      "Epoch 15/1000\n",
      "3321/3321 [==============================] - 0s - loss: 0.7658 - acc: 0.7362     \n",
      "Epoch 16/1000\n",
      "3321/3321 [==============================] - 1s - loss: 0.7751 - acc: 0.7341     \n",
      "Epoch 17/1000\n",
      "3321/3321 [==============================] - 1s - loss: 0.7636 - acc: 0.7290     \n",
      "Epoch 18/1000\n",
      "3321/3321 [==============================] - 1s - loss: 0.7547 - acc: 0.7386     \n",
      "Epoch 19/1000\n",
      "3321/3321 [==============================] - 1s - loss: 0.7670 - acc: 0.7341     \n",
      "Epoch 20/1000\n",
      "3321/3321 [==============================] - 1s - loss: 0.7242 - acc: 0.7386     \n",
      "Epoch 21/1000\n",
      "3321/3321 [==============================] - 1s - loss: 0.7711 - acc: 0.7266     \n",
      "Epoch 22/1000\n",
      "3321/3321 [==============================] - 1s - loss: 0.7370 - acc: 0.7374     \n",
      "Epoch 23/1000\n",
      "3321/3321 [==============================] - 1s - loss: 0.7270 - acc: 0.7471     \n",
      "Epoch 24/1000\n",
      "3321/3321 [==============================] - 1s - loss: 0.7162 - acc: 0.7483     \n",
      "Epoch 25/1000\n",
      "3321/3321 [==============================] - 1s - loss: 0.7440 - acc: 0.7407     \n",
      "Epoch 26/1000\n",
      "3321/3321 [==============================] - 1s - loss: 0.7424 - acc: 0.7453     \n",
      "Epoch 27/1000\n",
      "3321/3321 [==============================] - 1s - loss: 0.7215 - acc: 0.7377     \n",
      "Epoch 28/1000\n",
      "3321/3321 [==============================] - 1s - loss: 0.7389 - acc: 0.7398     \n",
      "Epoch 29/1000\n",
      "3321/3321 [==============================] - 0s - loss: 0.7447 - acc: 0.7341     \n",
      "Epoch 30/1000\n",
      "3321/3321 [==============================] - 0s - loss: 0.7683 - acc: 0.7329     \n",
      "Epoch 31/1000\n",
      "3321/3321 [==============================] - 1s - loss: 0.7357 - acc: 0.7404     \n",
      "Epoch 32/1000\n",
      "3321/3321 [==============================] - 1s - loss: 0.7459 - acc: 0.7395     \n",
      "Epoch 33/1000\n",
      "3321/3321 [==============================] - 1s - loss: 0.7521 - acc: 0.7380     \n",
      "Epoch 34/1000\n",
      "3321/3321 [==============================] - 1s - loss: 0.7569 - acc: 0.7483     \n",
      "Epoch 35/1000\n",
      "3321/3321 [==============================] - 1s - loss: 0.7328 - acc: 0.7410     \n",
      "Epoch 36/1000\n",
      "3321/3321 [==============================] - 1s - loss: 0.7368 - acc: 0.7335     \n",
      "Epoch 37/1000\n",
      "3321/3321 [==============================] - 1s - loss: 0.7166 - acc: 0.7525     \n",
      "Epoch 38/1000\n",
      "3321/3321 [==============================] - 1s - loss: 0.7289 - acc: 0.7492     \n",
      "Epoch 39/1000\n",
      "3321/3321 [==============================] - 1s - loss: 0.7278 - acc: 0.7528     \n",
      "Epoch 40/1000\n",
      "3321/3321 [==============================] - 1s - loss: 0.7345 - acc: 0.7504     \n",
      "Epoch 41/1000\n",
      "3321/3321 [==============================] - 1s - loss: 0.7280 - acc: 0.7410     \n",
      "Epoch 42/1000\n",
      "3321/3321 [==============================] - 0s - loss: 0.7456 - acc: 0.7401     \n",
      "Epoch 43/1000\n",
      "3321/3321 [==============================] - 1s - loss: 0.7320 - acc: 0.7398     \n",
      "Epoch 44/1000\n",
      "3321/3321 [==============================] - 0s - loss: 0.7415 - acc: 0.7438     \n",
      "Epoch 45/1000\n",
      "3321/3321 [==============================] - 0s - loss: 0.7460 - acc: 0.7453     \n",
      "Epoch 46/1000\n",
      "3321/3321 [==============================] - 0s - loss: 0.7529 - acc: 0.7459     \n",
      "Epoch 47/1000\n",
      "3321/3321 [==============================] - 1s - loss: 0.7593 - acc: 0.7395     \n",
      "Epoch 48/1000\n",
      "3321/3321 [==============================] - 1s - loss: 0.7414 - acc: 0.7428     \n",
      "Epoch 49/1000\n",
      "3321/3321 [==============================] - 0s - loss: 0.7435 - acc: 0.7332     \n",
      "Epoch 50/1000\n",
      "3321/3321 [==============================] - 0s - loss: 0.7618 - acc: 0.7401     \n",
      "Epoch 51/1000\n",
      "3321/3321 [==============================] - ETA: 0s - loss: 0.7347 - acc: 0.739 - 0s - loss: 0.7408 - acc: 0.7365     \n",
      "Epoch 52/1000\n",
      "3321/3321 [==============================] - 0s - loss: 0.7352 - acc: 0.7516     \n",
      "Epoch 53/1000\n",
      "3321/3321 [==============================] - 0s - loss: 0.7423 - acc: 0.7416     \n",
      "Epoch 54/1000\n",
      "3321/3321 [==============================] - 0s - loss: 0.7631 - acc: 0.7404     \n",
      "Epoch 55/1000\n",
      "3321/3321 [==============================] - 0s - loss: 0.7212 - acc: 0.7519     \n",
      "Epoch 56/1000\n",
      "3321/3321 [==============================] - 1s - loss: 0.7503 - acc: 0.7431     \n",
      "Epoch 57/1000\n",
      "3321/3321 [==============================] - 0s - loss: 0.7635 - acc: 0.7468     \n",
      "Epoch 58/1000\n",
      "3321/3321 [==============================] - 0s - loss: 0.7298 - acc: 0.7534     \n",
      "Epoch 59/1000\n",
      "3321/3321 [==============================] - 0s - loss: 0.7362 - acc: 0.7425     \n",
      "Epoch 60/1000\n",
      "3321/3321 [==============================] - 0s - loss: 0.7328 - acc: 0.7428     \n",
      "Epoch 61/1000\n",
      "3321/3321 [==============================] - 0s - loss: 0.7563 - acc: 0.7305     - ETA: 0s - loss: 0.73\n",
      "Epoch 62/1000\n",
      "3321/3321 [==============================] - 0s - loss: 0.7283 - acc: 0.7353     \n",
      "Epoch 63/1000\n",
      "3321/3321 [==============================] - 0s - loss: 0.7482 - acc: 0.7389     \n",
      "Epoch 64/1000\n",
      "3321/3321 [==============================] - 0s - loss: 0.7502 - acc: 0.7419     \n",
      "Epoch 65/1000\n",
      "3321/3321 [==============================] - 0s - loss: 0.7206 - acc: 0.7486     \n",
      "Epoch 66/1000\n",
      "3321/3321 [==============================] - 1s - loss: 0.7357 - acc: 0.7450     \n",
      "Epoch 67/1000\n",
      "3321/3321 [==============================] - 0s - loss: 0.7310 - acc: 0.7368     \n",
      "Epoch 68/1000\n",
      "3321/3321 [==============================] - 0s - loss: 0.7072 - acc: 0.7501     - ETA: 1s - loss: 0.6\n",
      "Epoch 69/1000\n",
      "3321/3321 [==============================] - 0s - loss: 0.7484 - acc: 0.7453     \n",
      "Epoch 70/1000\n",
      "3321/3321 [==============================] - 0s - loss: 0.7415 - acc: 0.7368     \n",
      "Epoch 71/1000\n",
      "3321/3321 [==============================] - 0s - loss: 0.7495 - acc: 0.7287     \n",
      "Epoch 72/1000\n",
      "3321/3321 [==============================] - 0s - loss: 0.7417 - acc: 0.7419     \n",
      "Epoch 73/1000\n",
      "3321/3321 [==============================] - 0s - loss: 0.7120 - acc: 0.7483     \n",
      "Epoch 74/1000\n",
      "3321/3321 [==============================] - 0s - loss: 0.7610 - acc: 0.7447     \n",
      "Epoch 75/1000\n",
      "3321/3321 [==============================] - 0s - loss: 0.7209 - acc: 0.7447     \n",
      "Epoch 76/1000\n",
      "3321/3321 [==============================] - 0s - loss: 0.7249 - acc: 0.7462     \n",
      "Epoch 77/1000\n",
      "3321/3321 [==============================] - 1s - loss: 0.7441 - acc: 0.7374     \n",
      "Epoch 78/1000\n",
      "3321/3321 [==============================] - 0s - loss: 0.7511 - acc: 0.7359     \n",
      "Epoch 79/1000\n",
      "3321/3321 [==============================] - 0s - loss: 0.7272 - acc: 0.7462     \n",
      "Epoch 80/1000\n",
      "3321/3321 [==============================] - 0s - loss: 0.7155 - acc: 0.7459     \n",
      "Epoch 81/1000\n",
      "3321/3321 [==============================] - 0s - loss: 0.7338 - acc: 0.7483     \n",
      "Epoch 82/1000\n",
      "3321/3321 [==============================] - 1s - loss: 0.7379 - acc: 0.7495     \n",
      "Epoch 83/1000\n",
      "3321/3321 [==============================] - 0s - loss: 0.7294 - acc: 0.7468     \n",
      "Epoch 84/1000\n",
      "3321/3321 [==============================] - 0s - loss: 0.7268 - acc: 0.7486     \n",
      "Epoch 85/1000\n",
      "3321/3321 [==============================] - 0s - loss: 0.7129 - acc: 0.7507     \n",
      "Epoch 86/1000\n",
      "3321/3321 [==============================] - 0s - loss: 0.7215 - acc: 0.7489     \n",
      "Epoch 87/1000\n",
      "3321/3321 [==============================] - 1s - loss: 0.7407 - acc: 0.7404     \n",
      "Epoch 88/1000\n",
      "3321/3321 [==============================] - 1s - loss: 0.7199 - acc: 0.7419     \n",
      "Epoch 89/1000\n",
      "3321/3321 [==============================] - 1s - loss: 0.7435 - acc: 0.7386     \n",
      "Epoch 90/1000\n",
      "3321/3321 [==============================] - 1s - loss: 0.7209 - acc: 0.7377     \n",
      "Epoch 91/1000\n",
      "3321/3321 [==============================] - 1s - loss: 0.7251 - acc: 0.7320     \n",
      "Epoch 92/1000\n",
      "3321/3321 [==============================] - 0s - loss: 0.6884 - acc: 0.7513     \n",
      "Epoch 93/1000\n",
      "3321/3321 [==============================] - 0s - loss: 0.7258 - acc: 0.7389     \n",
      "Epoch 94/1000\n",
      "3321/3321 [==============================] - 1s - loss: 0.7252 - acc: 0.7377     \n",
      "Epoch 95/1000\n",
      "3321/3321 [==============================] - 0s - loss: 0.7360 - acc: 0.7389     \n",
      "Epoch 96/1000\n",
      "3321/3321 [==============================] - 0s - loss: 0.7237 - acc: 0.7471     \n",
      "Epoch 97/1000\n",
      "3321/3321 [==============================] - ETA: 0s - loss: 0.7240 - acc: 0.742 - 1s - loss: 0.7191 - acc: 0.7444     \n",
      "Epoch 98/1000\n",
      "3321/3321 [==============================] - 1s - loss: 0.7273 - acc: 0.7407     \n",
      "Epoch 99/1000\n",
      "3321/3321 [==============================] - 1s - loss: 0.7061 - acc: 0.7543     \n",
      "Epoch 100/1000\n",
      "3321/3321 [==============================] - 0s - loss: 0.7285 - acc: 0.7392     \n",
      "Epoch 101/1000\n",
      "3321/3321 [==============================] - 1s - loss: 0.7232 - acc: 0.7546     \n",
      "Epoch 102/1000\n",
      "3321/3321 [==============================] - 1s - loss: 0.7485 - acc: 0.7335     \n",
      "Epoch 103/1000\n",
      "3321/3321 [==============================] - 1s - loss: 0.7174 - acc: 0.7389     \n",
      "Epoch 104/1000\n",
      "3321/3321 [==============================] - 0s - loss: 0.7217 - acc: 0.7444     \n",
      "Epoch 105/1000\n",
      "3321/3321 [==============================] - 1s - loss: 0.7207 - acc: 0.7480     \n",
      "Epoch 106/1000\n",
      "3321/3321 [==============================] - 1s - loss: 0.7378 - acc: 0.7413     \n",
      "Epoch 107/1000\n",
      "3321/3321 [==============================] - 1s - loss: 0.7558 - acc: 0.7335     \n",
      "Epoch 108/1000\n",
      "3321/3321 [==============================] - 1s - loss: 0.7466 - acc: 0.7413     \n",
      "Epoch 109/1000\n",
      "3321/3321 [==============================] - 1s - loss: 0.6980 - acc: 0.7510     \n",
      "Epoch 110/1000\n",
      "3321/3321 [==============================] - 1s - loss: 0.7266 - acc: 0.7359     \n",
      "Epoch 111/1000\n",
      "3321/3321 [==============================] - 1s - loss: 0.7175 - acc: 0.7428     \n",
      "Epoch 112/1000\n",
      "3321/3321 [==============================] - 1s - loss: 0.7214 - acc: 0.7335     \n",
      "Epoch 113/1000\n",
      "3321/3321 [==============================] - 1s - loss: 0.7188 - acc: 0.7462     \n",
      "Epoch 114/1000\n",
      "3321/3321 [==============================] - 1s - loss: 0.7324 - acc: 0.7413     \n",
      "Epoch 115/1000\n",
      "3321/3321 [==============================] - 1s - loss: 0.7415 - acc: 0.7438     \n",
      "Epoch 116/1000\n",
      "3321/3321 [==============================] - 1s - loss: 0.7184 - acc: 0.7447     \n",
      "Epoch 117/1000\n",
      "3321/3321 [==============================] - 1s - loss: 0.7319 - acc: 0.7431     \n",
      "Epoch 118/1000\n",
      "3321/3321 [==============================] - 1s - loss: 0.7365 - acc: 0.7431     \n",
      "Epoch 119/1000\n",
      "3321/3321 [==============================] - 0s - loss: 0.7046 - acc: 0.7450     \n",
      "Epoch 120/1000\n",
      "3321/3321 [==============================] - 0s - loss: 0.7062 - acc: 0.7549     \n",
      "Epoch 121/1000\n",
      "3321/3321 [==============================] - 0s - loss: 0.7383 - acc: 0.7383     \n",
      "Epoch 122/1000\n",
      "3321/3321 [==============================] - 0s - loss: 0.7345 - acc: 0.7435     \n",
      "Epoch 123/1000\n",
      "3321/3321 [==============================] - 0s - loss: 0.7418 - acc: 0.7386     \n",
      "Epoch 124/1000\n",
      "3321/3321 [==============================] - 0s - loss: 0.7458 - acc: 0.7371     \n",
      "Epoch 125/1000\n",
      "3321/3321 [==============================] - 0s - loss: 0.7364 - acc: 0.7413     \n",
      "Epoch 126/1000\n",
      "3321/3321 [==============================] - 0s - loss: 0.7531 - acc: 0.7278     \n",
      "Epoch 127/1000\n",
      "3321/3321 [==============================] - 0s - loss: 0.7287 - acc: 0.7465     \n",
      "Epoch 128/1000\n",
      "3321/3321 [==============================] - 0s - loss: 0.7590 - acc: 0.7302     \n",
      "Epoch 129/1000\n",
      "3321/3321 [==============================] - 0s - loss: 0.7583 - acc: 0.7435     \n",
      "Epoch 130/1000\n",
      "3321/3321 [==============================] - 0s - loss: 0.7150 - acc: 0.7471     \n",
      "Epoch 131/1000\n",
      "3321/3321 [==============================] - 0s - loss: 0.7602 - acc: 0.7350     \n",
      "Epoch 132/1000\n",
      "3321/3321 [==============================] - 0s - loss: 0.7337 - acc: 0.7410     \n",
      "Epoch 133/1000\n",
      "3321/3321 [==============================] - 0s - loss: 0.7357 - acc: 0.7444     \n",
      "Epoch 134/1000\n",
      "3321/3321 [==============================] - 1s - loss: 0.7607 - acc: 0.7350     \n",
      "Epoch 135/1000\n",
      "3321/3321 [==============================] - 1s - loss: 0.7217 - acc: 0.7401     \n",
      "Epoch 136/1000\n",
      "3321/3321 [==============================] - 1s - loss: 0.7165 - acc: 0.7471     \n",
      "Epoch 137/1000\n",
      "3321/3321 [==============================] - 1s - loss: 0.7312 - acc: 0.7335     \n",
      "Epoch 138/1000\n",
      "3321/3321 [==============================] - 1s - loss: 0.7066 - acc: 0.7525     \n",
      "Epoch 139/1000\n",
      "3321/3321 [==============================] - 0s - loss: 0.7167 - acc: 0.7459     \n",
      "Epoch 140/1000\n",
      "3321/3321 [==============================] - 0s - loss: 0.7331 - acc: 0.7347     \n",
      "Epoch 141/1000\n",
      "3321/3321 [==============================] - 0s - loss: 0.7479 - acc: 0.7344     \n",
      "Epoch 142/1000\n",
      "3321/3321 [==============================] - 0s - loss: 0.7257 - acc: 0.7483     \n",
      "Epoch 143/1000\n",
      "3321/3321 [==============================] - 0s - loss: 0.7304 - acc: 0.7425     \n",
      "Epoch 00142: early stopping\n",
      "Elapsed time: 154.23 sec\n",
      "Saved model and weights to disk\n",
      "CPU times: user 5min 57s, sys: 1min 37s, total: 7min 34s\n",
      "Wall time: 2min 34s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "start_time = time.time()\n",
    "#estimator.fit(svd_train, y_ind, validation_split=0.05)\n",
    "estimator.fit(svd_train, y_ind, batch_size=batch_n, epochs=EPOCHS_N*10, callbacks=[weight_save_callback, early_stopping])\n",
    "end_time = time.time()\n",
    "print(\"Elapsed time: {:.2f} sec\".format(end_time-start_time))\n",
    "try: \n",
    "    save_model_to_json(estimator, model_path)\n",
    "    print(\"Saved model and weights to disk\")\n",
    "except Exception as e:\n",
    "    print(e)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Display model architecture\n",
    "from IPython.display import SVG\n",
    "from keras.utils.vis_utils import model_to_dot\n",
    "\n",
    "SVG(model_to_dot(model_hypothesis()).create(prog='dot', format='svg'))\n",
    "\n",
    "# Save to .png\n",
    "from keras.utils import plot_model\n",
    "plot_model(model_hypothesis(), to_file='model_hypothesis1.png')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "5184/5668 [==========================>...] - ETA: 0s"
     ]
    }
   ],
   "source": [
    "y_pred = estimator.predict_proba(svd_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "submission = pd.DataFrame(y_pred)\n",
    "submission['id'] = test_index\n",
    "submission.columns = ['class1', 'class2', 'class3', 'class4', 'class5', 'class6', 'class7', 'class8', 'class9', 'id']\n",
    "submission.to_csv(data_directory + \"/output/submission_\" + str(int(time.time())) + \".csv\",index=False)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python [conda env:W207-final]",
   "language": "python",
   "name": "conda-env-W207-final-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}