Chap07-Advanced_deep-learning_best_practices/hyperas-test.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 7.3.2 Hyperas를 이용한 하이퍼파라미터 최적화 구현하기"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "- https://github.com/maxpumperla/hyperas"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "# !pip install hyperas\n",
    "\n",
    "import numpy as np\n",
    "\n",
    "from hyperopt import Trials, STATUS_OK, tpe\n",
    "from keras.datasets import mnist\n",
    "from keras import layers, models\n",
    "\n",
    "from hyperas import optim\n",
    "from hyperas.distributions import choice, uniform"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "# create data() function\n",
    "\n",
    "def data():\n",
    "    (train_x, train_y), (test_x, test_y) = mnist.load_data()\n",
    "    train_x = train_x.astype('float32').reshape([60000, -1]) / 255.\n",
    "    test_x = test_x.astype('float32').reshape([10000, -1]) / 255. \n",
    "    \n",
    "    return train_x, train_y, test_x, test_y\n",
    "\n",
    "\n",
    "# create model function\n",
    "def mnist_model(train_x, train_y, test_x, test_y):\n",
    "    model = models.Sequential()\n",
    "    model.add(layers.Dense(256, activation='relu', input_shape=(784,)))\n",
    "    model.add(layers.Dropout({{uniform(0, 1)}}))\n",
    "    model.add(layers.Dense({{choice([256, 128])}}, activation='relu'))\n",
    "    model.add(layers.Dense(10, activation='softmax'))\n",
    "    model.compile(optimizer={{choice(['rmsprop', 'adam'])}}, \n",
    "                  loss='sparse_categorical_crossentropy',\n",
    "                  metrics=['acc'])\n",
    "    \n",
    "    result = model.fit(train_x, train_y, \n",
    "                       batch_size={{choice([64, 128])}}, \n",
    "                       epochs=2,\n",
    "                       verbose=2,\n",
    "                       validation_split=0.2)\n",
    "    #get the highest validation accuracy of the training epochs\n",
    "    validation_acc = np.amax(result.history['val_acc'])\n",
    "    validation_loss = np.amin(result.history['val_loss'])\n",
    "    print('Best validation acc of epoch:', validation_acc)\n",
    "    return {'loss': validation_loss, 'status': STATUS_OK, 'model': model}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      ">>> Imports:\n",
      "#coding=utf-8\n",
      "\n",
      "try:\n",
      "    import numpy as np\n",
      "except:\n",
      "    pass\n",
      "\n",
      "try:\n",
      "    from hyperopt import Trials, STATUS_OK, tpe\n",
      "except:\n",
      "    pass\n",
      "\n",
      "try:\n",
      "    from keras.datasets import mnist\n",
      "except:\n",
      "    pass\n",
      "\n",
      "try:\n",
      "    from keras import layers, models\n",
      "except:\n",
      "    pass\n",
      "\n",
      "try:\n",
      "    from hyperas import optim\n",
      "except:\n",
      "    pass\n",
      "\n",
      "try:\n",
      "    from hyperas.distributions import choice, uniform\n",
      "except:\n",
      "    pass\n",
      "\n",
      ">>> Hyperas search space:\n",
      "\n",
      "def get_space():\n",
      "    return {\n",
      "        'Dropout': hp.uniform('Dropout', 0, 1),\n",
      "        'Dense': hp.choice('Dense', [256, 128]),\n",
      "        'optimizer': hp.choice('optimizer', ['rmsprop', 'adam']),\n",
      "        'batch_size': hp.choice('batch_size', [64, 128]),\n",
      "    }\n",
      "\n",
      ">>> Data\n",
      "  1: \n",
      "  2: (train_x, train_y), (test_x, test_y) = mnist.load_data()\n",
      "  3: train_x = train_x.astype('float32').reshape([60000, -1]) / 255.\n",
      "  4: test_x = test_x.astype('float32').reshape([10000, -1]) / 255. \n",
      "  5: \n",
      "  6: \n",
      "  7: \n",
      "  8: \n",
      ">>> Resulting replaced keras model:\n",
      "\n",
      "  1: def keras_fmin_fnct(space):\n",
      "  2: \n",
      "  3:     model = models.Sequential()\n",
      "  4:     model.add(layers.Dense(256, activation='relu', input_shape=(784,)))\n",
      "  5:     model.add(layers.Dropout(space['Dropout']))\n",
      "  6:     model.add(layers.Dense(space['Dense'], activation='relu'))\n",
      "  7:     model.add(layers.Dense(10, activation='softmax'))\n",
      "  8:     model.compile(optimizer=space['optimizer'], \n",
      "  9:                   loss='sparse_categorical_crossentropy',\n",
      " 10:                   metrics=['acc'])\n",
      " 11:     \n",
      " 12:     result = model.fit(train_x, train_y, \n",
      " 13:                        batch_size=space['batch_size'], \n",
      " 14:                        epochs=2,\n",
      " 15:                        verbose=2,\n",
      " 16:                        validation_split=0.2)\n",
      " 17:     #get the highest validation accuracy of the training epochs\n",
      " 18:     validation_acc = np.amax(result.history['val_acc'])\n",
      " 19:     validation_loss = np.amin(result.history['val_loss'])\n",
      " 20:     print('Best validation acc of epoch:', validation_acc)\n",
      " 21:     return {'loss': validation_loss, 'status': STATUS_OK, 'model': model}\n",
      " 22: \n",
      "Train on 48000 samples, validate on 12000 samples\n",
      "Epoch 1/2\n",
      " - 1s - loss: 0.4850 - acc: 0.8522 - val_loss: 0.1865 - val_acc: 0.9462\n",
      "Epoch 2/2\n",
      " - 1s - loss: 0.2650 - acc: 0.9201 - val_loss: 0.1416 - val_acc: 0.9588\n",
      "Best validation acc of epoch: 0.9588333334922791\n",
      "Train on 48000 samples, validate on 12000 samples\n",
      "Epoch 1/2\n",
      " - 2s - loss: 1.0466 - acc: 0.6411 - val_loss: 0.3284 - val_acc: 0.9087\n",
      "Epoch 2/2\n",
      " - 2s - loss: 0.7334 - acc: 0.7676 - val_loss: 0.2802 - val_acc: 0.9215\n",
      "Best validation acc of epoch: 0.9215\n",
      "Train on 48000 samples, validate on 12000 samples\n",
      "Epoch 1/2\n",
      " - 2s - loss: 0.3134 - acc: 0.9064 - val_loss: 0.1349 - val_acc: 0.9593\n",
      "Epoch 2/2\n",
      " - 2s - loss: 0.1375 - acc: 0.9588 - val_loss: 0.0979 - val_acc: 0.9712\n",
      "Best validation acc of epoch: 0.97125\n",
      "Train on 48000 samples, validate on 12000 samples\n",
      "Epoch 1/2\n",
      " - 1s - loss: 0.4219 - acc: 0.8735 - val_loss: 0.1768 - val_acc: 0.9471\n",
      "Epoch 2/2\n",
      " - 1s - loss: 0.1902 - acc: 0.9428 - val_loss: 0.1209 - val_acc: 0.9635\n",
      "Best validation acc of epoch: 0.9635\n",
      "Train on 48000 samples, validate on 12000 samples\n",
      "Epoch 1/2\n",
      " - 2s - loss: 0.4136 - acc: 0.8752 - val_loss: 0.1603 - val_acc: 0.9506\n",
      "Epoch 2/2\n",
      " - 2s - loss: 0.2145 - acc: 0.9341 - val_loss: 0.1295 - val_acc: 0.9600\n",
      "Best validation acc of epoch: 0.96\n",
      "CPU times: user 30.9 s, sys: 2.8 s, total: 33.7 s\n",
      "Wall time: 19.2 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "notebook_name = 'hyperas-test'\n",
    "\n",
    "best_run, best_model = optim.minimize(model=mnist_model,\n",
    "                                      data=data,\n",
    "                                      algo=tpe.suggest,\n",
    "                                      max_evals=5,\n",
    "                                      trials=Trials(),\n",
    "                                      notebook_name=notebook_name)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "10000/10000 [==============================] - 0s 26us/step\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[0.09437107192506082, 0.9703]"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_x, train_y, test_x, test_y = data()\n",
    "\n",
    "best_model.evaluate(test_x, test_y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'Dense': 1, 'Dropout': 0.288662535902546, 'batch_size': 0, 'optimizer': 1}"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "best_run"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "tensorflow",
   "language": "python",
   "name": "tensorflow"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}