ga_nn.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Genetic Algorithm on Keras"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### MNIST training on Keras with Genetic Algorithm\n",
    "* This notebook runs MNIST training on Keras using Genetic Algorithm to find the best hyper parameters.\n",
    "* The MNIST model here is just a simple one with one input layer, one hidden layer and one output layer, without convolution.\n",
    "* Hyperparameters of the model include the followings:\n",
    "* - output shape of the first layer\n",
    "* - dropout rate of the first layer\n",
    "* - output shape of the second layer\n",
    "* - dropout rate of the second layer\n",
    "* - use batchnormalization or not\n",
    "* - batch size\n",
    "* - number of epochs\n",
    "* - validation rate\n",
    "* I used deap to run GA."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Import libraries"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "from deap import base, creator, tools, algorithms\n",
    "\n",
    "import numpy as np\n",
    "import pandas as pds\n",
    "import random\n",
    "from keras.layers import Activation, Dropout, BatchNormalization, Dense\n",
    "from keras.models import Sequential\n",
    "from keras.datasets import mnist\n",
    "from keras.metrics import categorical_crossentropy\n",
    "from keras.utils import np_utils\n",
    "from keras.optimizers import Adam\n",
    "from keras.callbacks import EarlyStopping"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Define MNIST model\n",
    "* includes data loading function, training function, fit function and evaluation function "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# MNIST class\n",
    "class MNIST():\n",
    "    def __init__(self,\n",
    "                 l1_out=512, \n",
    "                 l2_out=512, \n",
    "                 l1_drop=0.2, \n",
    "                 l2_drop=0.2, \n",
    "                 bn1=0,\n",
    "                 bn2=0,\n",
    "                 batch_size=100, \n",
    "                 epochs=10, \n",
    "                 validation_split=0.1):\n",
    "        self.l1_out = l1_out\n",
    "        self.l2_out = l2_out\n",
    "        self.l1_drop = l1_drop\n",
    "        self.l2_drop = l2_drop\n",
    "        self.bn1 = bn1\n",
    "        self.bn2 = bn2\n",
    "        self.batch_size = batch_size\n",
    "        self.epochs = epochs\n",
    "        self.validation_split = validation_split\n",
    "        self.__x_train, self.__x_test, self.__y_train, self.__y_test = self.mnist_data()\n",
    "        self.__model = self.mnist_model()\n",
    "        params = \"\"\"\n",
    "        l1_out:\\t{0}\n",
    "        l2_out:\\t{1}\n",
    "        l1_drop:\\t{2}\n",
    "        l2_drop:\\t{3}\n",
    "        bn1:\\t{4}\n",
    "        bn2:\\t{5}\n",
    "        batch_size:\\t{6}\n",
    "        epochs:\\t{7}\n",
    "        validation_split:\\t{8}\n",
    "        \"\"\".format(self.l1_out, self.l2_out,\n",
    "                   self.l1_drop, self.l2_drop,\n",
    "                   self.bn1, self.bn2,\n",
    "                   self.batch_size, self.epochs,\n",
    "                   self.validation_split)\n",
    "        print(params)\n",
    "        \n",
    "    # load mnist data from keras dataset\n",
    "    def mnist_data(self):\n",
    "        (X_train, y_train), (X_test, y_test) = mnist.load_data()\n",
    "        X_train = X_train.reshape(60000, 784)\n",
    "        X_test = X_test.reshape(10000, 784)\n",
    "\n",
    "        X_train = X_train.astype('float32')\n",
    "        X_test = X_test.astype('float32')\n",
    "        X_train /= 255\n",
    "        X_test /= 255\n",
    "\n",
    "        Y_train = np_utils.to_categorical(y_train, 10)\n",
    "        Y_test = np_utils.to_categorical(y_test, 10)\n",
    "        return X_train, X_test, Y_train, Y_test\n",
    "    \n",
    "    # mnist model\n",
    "    def mnist_model(self):\n",
    "        model = Sequential()\n",
    "        model.add(Dense(self.l1_out, input_shape=(784,)))\n",
    "        if self.bn1 == 0:\n",
    "            model.add(BatchNormalization())\n",
    "        model.add(Activation('relu'))\n",
    "        model.add(Dropout(self.l1_drop))\n",
    "        model.add(Dense(self.l2_out))\n",
    "        if self.bn2 == 0:\n",
    "            model.add(BatchNormalization())\n",
    "        model.add(Activation('relu'))\n",
    "        model.add(Dropout(self.l2_drop))\n",
    "        model.add(Dense(10))\n",
    "        model.add(Activation('softmax'))\n",
    "        model.compile(loss='categorical_crossentropy',\n",
    "                      optimizer=Adam(),\n",
    "                      metrics=['accuracy'])\n",
    "\n",
    "        return model\n",
    "    \n",
    "    # fit mnist model\n",
    "    def mnist_fit(self):\n",
    "        early_stopping = EarlyStopping(patience=0, verbose=1)\n",
    "        \n",
    "        self.__model.fit(self.__x_train, self.__y_train,\n",
    "                       batch_size=self.batch_size,\n",
    "                       epochs=self.epochs,\n",
    "                       verbose=0,\n",
    "                       validation_split=self.validation_split,\n",
    "                       callbacks=[early_stopping])\n",
    "    \n",
    "    # evaluate mnist model\n",
    "    def mnist_evaluate(self):\n",
    "        self.mnist_fit()\n",
    "        \n",
    "        evaluation = self.__model.evaluate(self.__x_test, self.__y_test, batch_size=self.batch_size, verbose=0)\n",
    "        return evaluation"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Runner function for the MNIST model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# function to run mnist class\n",
    "def run_mnist(bounds):\n",
    "    _mnist = MNIST(l1_out=bounds[0], \n",
    "                   l2_out=bounds[1], \n",
    "                   l1_drop=bounds[2], \n",
    "                   l2_drop=bounds[3], \n",
    "                   bn1=bounds[4],\n",
    "                   bn2=bounds[5],\n",
    "                   batch_size=bounds[6],\n",
    "                   epochs=bounds[7], \n",
    "                   validation_split=bounds[8])\n",
    "    mnist_evaluation = _mnist.mnist_evaluate()\n",
    "    print(mnist_evaluation)\n",
    "    return mnist_evaluation[0],"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Defining Genetic Algorithm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# creator\n",
    "creator.create('FitnessMax', base.Fitness, weights = (-1.0,))\n",
    "creator.create('Individual', list , fitness = creator.FitnessMax)\n",
    "\n",
    "# defining attributes for individual\n",
    "toolbox = base.Toolbox()\n",
    "\n",
    "# layer outputs\n",
    "toolbox.register(\"l1_out\", random.choice, (64, 128, 256, 512, 1024))\n",
    "toolbox.register(\"l2_out\", random.choice, (64, 128, 256, 512, 1024))\n",
    "# dropout late\n",
    "toolbox.register(\"l1_drop\", random.uniform, 0.0, 0.3)\n",
    "toolbox.register(\"l2_drop\", random.uniform, 0.0, 0.3)\n",
    "# batchnormalization\n",
    "toolbox.register(\"bn1\", random.randint, 0, 1)\n",
    "toolbox.register(\"bn2\", random.randint, 0, 1)\n",
    "# training\n",
    "toolbox.register(\"batch_size\", random.choice, (10, 100, 500))\n",
    "toolbox.register(\"epochs\", random.choice, (5, 10, 20))\n",
    "toolbox.register(\"validation_split\", random.uniform, 0.0, 0.3)\n",
    "\n",
    "# register attributes to individual\n",
    "toolbox.register('individual', tools.initCycle, creator.Individual, \n",
    "                 (toolbox.l1_out, toolbox.l2_out, \n",
    "                  toolbox.l1_drop, toolbox.l2_drop,\n",
    "                  toolbox.bn1, toolbox.bn2, \n",
    "                  toolbox.batch_size, toolbox.epochs, toolbox.validation_split), \n",
    "                  n = 1)\n",
    "# individual to population\n",
    "toolbox.register('population', tools.initRepeat, list, toolbox.individual)\n",
    "\n",
    "# evolution\n",
    "toolbox.register('mate', tools.cxTwoPoint)\n",
    "toolbox.register('mutate', tools.mutFlipBit, indpb = 0.05)\n",
    "toolbox.register('select', tools.selTournament, tournsize=3)\n",
    "toolbox.register('evaluate', run_mnist)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true,
    "scrolled": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Function for running hyper parameter optimization on Genetic Algorithm "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def genAlg(population=5, CXPB=0.5, MUTPB=0.2, NGEN=5):\n",
    "    random.seed(64)\n",
    "    pop = toolbox.population(n=population)\n",
    "    \n",
    "    print(\"Start of evolution\")\n",
    "\n",
    "    fitnesses = list(map(toolbox.evaluate, pop))\n",
    "    for ind, fit in zip(pop, fitnesses):\n",
    "        ind.fitness.values = fit\n",
    "    print(\"  Evaluated %i individuals\" % len(pop))\n",
    "    \n",
    "    for g in range(NGEN):\n",
    "        print(\"-- Generation %i --\" % g)\n",
    "\n",
    "        offspring = toolbox.select(pop, len(pop))\n",
    "        offspring = list(map(toolbox.clone, offspring))\n",
    "\n",
    "        for child1, child2 in zip(offspring[::2], offspring[1::2]):\n",
    "            if random.random() < CXPB:\n",
    "                print(\"mate\")\n",
    "                toolbox.mate(child1, child2)\n",
    "                del child1.fitness.values\n",
    "                del child2.fitness.values\n",
    "\n",
    "        for mutant in offspring:\n",
    "            if random.random() < MUTPB:\n",
    "                print(\"mutate\")\n",
    "                toolbox.mutate(mutant)\n",
    "                del mutant.fitness.values\n",
    "\n",
    "        try:\n",
    "            invalid_ind = [ind for ind in offspring if not ind.fitness.valid]\n",
    "            fitnesses = map(toolbox.evaluate, invalid_ind)\n",
    "            for ind, fit in zip(invalid_ind, fitnesses):\n",
    "                ind.fitness.values = fit\n",
    "        except AssertionError:\n",
    "            pass\n",
    "                \n",
    "        print(\"  Evaluated %i individuals\" % len(invalid_ind))\n",
    "\n",
    "        pop[:] = offspring\n",
    "\n",
    "        try:\n",
    "            fits = [ind.fitness.values[0] for ind in pop]\n",
    "\n",
    "            length = len(pop)\n",
    "            mean = sum(fits) / length\n",
    "            sum2 = sum(x*x for x in fits)\n",
    "            std = abs(sum2 / length - mean**2)**0.5\n",
    "\n",
    "            print(\"  Min %s\" % min(fits))\n",
    "            print(\"  Max %s\" % max(fits))\n",
    "            print(\"  Avg %s\" % mean)\n",
    "            print(\"  Std %s\" % std)\n",
    "        except IndexError:\n",
    "            pass\n",
    "\n",
    "    print(\"-- End of (successful) evolution --\")\n",
    "\n",
    "    best_ind = tools.selBest(pop, 1)[0]\n",
    "    print(\"Best individual is %s, %s\" % (best_ind, best_ind.fitness.values))\n",
    "    return best_ind"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Start of evolution\n",
      "\n",
      "        l1_out:\t512\n",
      "        l2_out:\t64\n",
      "        l1_drop:\t0.18906571629566554\n",
      "        l2_drop:\t0.12118829087054188\n",
      "        bn1:\t0\n",
      "        bn2:\t0\n",
      "        batch_size:\t100\n",
      "        epochs:\t20\n",
      "        validation_split:\t0.27380697683392835\n",
      "        \n",
      "Epoch 00003: early stopping\n",
      "[0.082229894990450705, 0.97410000741481784]\n",
      "\n",
      "        l1_out:\t128\n",
      "        l2_out:\t128\n",
      "        l1_drop:\t0.023943619120116687\n",
      "        l2_drop:\t0.12502013021715647\n",
      "        bn1:\t0\n",
      "        bn2:\t0\n",
      "        batch_size:\t10\n",
      "        epochs:\t10\n",
      "        validation_split:\t0.20383943374115368\n",
      "        \n",
      "Epoch 00004: early stopping\n",
      "[0.078112404181742021, 0.97649999552965161]\n",
      "\n",
      "        l1_out:\t64\n",
      "        l2_out:\t64\n",
      "        l1_drop:\t0.052369065073309795\n",
      "        l2_drop:\t0.17600891153302792\n",
      "        bn1:\t1\n",
      "        bn2:\t0\n",
      "        batch_size:\t500\n",
      "        epochs:\t5\n",
      "        validation_split:\t0.026646503392500687\n",
      "        \n",
      "[0.1098207978066057, 0.96820000112056737]\n",
      "\n",
      "        l1_out:\t64\n",
      "        l2_out:\t512\n",
      "        l1_drop:\t0.11910387551405677\n",
      "        l2_drop:\t0.06412721938460672\n",
      "        bn1:\t0\n",
      "        bn2:\t1\n",
      "        batch_size:\t100\n",
      "        epochs:\t20\n",
      "        validation_split:\t0.09210574048914517\n",
      "        \n",
      "Epoch 00006: early stopping\n",
      "[0.081265506267081941, 0.97530000627040858]\n",
      "\n",
      "        l1_out:\t256\n",
      "        l2_out:\t512\n",
      "        l1_drop:\t0.15575202668971602\n",
      "        l2_drop:\t0.015022987876030168\n",
      "        bn1:\t0\n",
      "        bn2:\t1\n",
      "        batch_size:\t10\n",
      "        epochs:\t5\n",
      "        validation_split:\t0.06589604628677845\n",
      "        \n",
      "Epoch 00002: early stopping\n",
      "[0.085461119820763085, 0.97449999570846557]\n",
      "  Evaluated 5 individuals\n",
      "-- Generation 0 --\n",
      "mate\n",
      "mutate\n",
      "mutate\n",
      "\n",
      "        l1_out:\t128\n",
      "        l2_out:\t128\n",
      "        l1_drop:\t0.0\n",
      "        l2_drop:\t0.12502013021715647\n",
      "        bn1:\t0\n",
      "        bn2:\t0\n",
      "        batch_size:\t100\n",
      "        epochs:\t20\n",
      "        validation_split:\t0.20383943374115368\n",
      "        \n",
      "Epoch 00005: early stopping\n",
      "[0.091630371784267486, 0.9723000049591064]\n",
      "\n",
      "        l1_out:\t512\n",
      "        l2_out:\t64\n",
      "        l1_drop:\t0.18906571629566554\n",
      "        l2_drop:\t0.12118829087054188\n",
      "        bn1:\t0\n",
      "        bn2:\t0\n",
      "        batch_size:\t10\n",
      "        epochs:\t10\n",
      "        validation_split:\t0.27380697683392835\n",
      "        \n",
      "Epoch 00006: early stopping\n",
      "[0.076537680939287381, 0.97779999595880507]\n",
      "\n",
      "        l1_out:\t128\n",
      "        l2_out:\t128\n",
      "        l1_drop:\t0.023943619120116687\n",
      "        l2_drop:\t0.12502013021715647\n",
      "        bn1:\t0\n",
      "        bn2:\t0\n",
      "        batch_size:\t10\n",
      "        epochs:\t10\n",
      "        validation_split:\t0.20383943374115368\n",
      "        \n",
      "Epoch 00005: early stopping\n",
      "[0.072652313994942236, 0.9798999963402748]\n",
      "  Evaluated 3 individuals\n",
      "  Min 0.0726523139949\n",
      "  Max 0.0916303717843\n",
      "  Avg 0.0794090350164\n",
      "  Std 0.0064283983796\n",
      "-- Generation 1 --\n",
      "mutate\n",
      "\n",
      "        l1_out:\t128\n",
      "        l2_out:\t128\n",
      "        l1_drop:\t0.023943619120116687\n",
      "        l2_drop:\t0.12502013021715647\n",
      "        bn1:\t0\n",
      "        bn2:\t0\n",
      "        batch_size:\t10\n",
      "        epochs:\t0\n",
      "        validation_split:\t0.20383943374115368\n",
      "        \n",
      "[2.343395842552185, 0.089600001618266104]\n",
      "  Evaluated 1 individuals\n",
      "  Min 0.0726523139949\n",
      "  Max 2.34339584255\n",
      "  Avg 0.529447184521\n",
      "  Std 0.906976119699\n",
      "-- Generation 2 --\n",
      "mate\n",
      "mutate\n",
      "\n",
      "        l1_out:\t512\n",
      "        l2_out:\t64\n",
      "        l1_drop:\t0.18906571629566554\n",
      "        l2_drop:\t0.12502013021715647\n",
      "        bn1:\t0\n",
      "        bn2:\t0\n",
      "        batch_size:\t10\n",
      "        epochs:\t10\n",
      "        validation_split:\t0.27380697683392835\n",
      "        \n",
      "Epoch 00007: early stopping\n",
      "[0.072084434422857155, 0.97799999636411672]\n",
      "\n",
      "        l1_out:\t128\n",
      "        l2_out:\t128\n",
      "        l1_drop:\t0.023943619120116687\n",
      "        l2_drop:\t0.12118829087054188\n",
      "        bn1:\t0\n",
      "        bn2:\t0\n",
      "        batch_size:\t10\n",
      "        epochs:\t10\n",
      "        validation_split:\t0.20383943374115368\n",
      "        \n",
      "Epoch 00003: early stopping\n",
      "[0.088837052826551369, 0.97289999568462371]\n",
      "\n",
      "        l1_out:\t128\n",
      "        l2_out:\t128\n",
      "        l1_drop:\t0.023943619120116687\n",
      "        l2_drop:\t0.12502013021715647\n",
      "        bn1:\t0\n",
      "        bn2:\t0\n",
      "        batch_size:\t10\n",
      "        epochs:\t10\n",
      "        validation_split:\t0.20383943374115368\n",
      "        \n",
      "Epoch 00006: early stopping\n",
      "[0.071842810310507044, 0.97929999607801432]\n",
      "  Evaluated 3 individuals\n",
      "  Min 0.0718428103105\n",
      "  Max 0.0888370528266\n",
      "  Avg 0.07561378511\n",
      "  Std 0.00661924464944\n",
      "-- Generation 3 --\n",
      "  Evaluated 0 individuals\n",
      "  Min 0.0718428103105\n",
      "  Max 0.0726523139949\n",
      "  Avg 0.0721666117843\n",
      "  Std 0.000396574194354\n",
      "-- Generation 4 --\n",
      "mate\n",
      "mutate\n",
      "\n",
      "        l1_out:\t128\n",
      "        l2_out:\t128\n",
      "        l1_drop:\t0.023943619120116687\n",
      "        l2_drop:\t0.12502013021715647\n",
      "        bn1:\t0\n",
      "        bn2:\t0\n",
      "        batch_size:\t10\n",
      "        epochs:\t10\n",
      "        validation_split:\t0.20383943374115368\n",
      "        \n",
      "Epoch 00004: early stopping\n",
      "[0.084735829577679403, 0.97519999605417251]\n",
      "  Evaluated 2 individuals\n",
      "-- Generation 5 --\n",
      "mate\n",
      "\n",
      "        l1_out:\t128\n",
      "        l2_out:\t128\n",
      "        l1_drop:\t0.023943619120116687\n",
      "        l2_drop:\t0.12502013021715647\n",
      "        bn1:\t0\n",
      "        bn2:\t0\n",
      "        batch_size:\t10\n",
      "        epochs:\t10\n",
      "        validation_split:\t0.20383943374115368\n",
      "        \n",
      "Epoch 00005: early stopping\n",
      "[0.077699606033085275, 0.97769999611377711]\n",
      "\n",
      "        l1_out:\t128\n",
      "        l2_out:\t128\n",
      "        l1_drop:\t0.023943619120116687\n",
      "        l2_drop:\t0.12502013021715647\n",
      "        bn1:\t0\n",
      "        bn2:\t0\n",
      "        batch_size:\t10\n",
      "        epochs:\t10\n",
      "        validation_split:\t0.20383943374115368\n",
      "        \n",
      "Epoch 00006: early stopping\n",
      "[0.069713935270881849, 0.97809999638795853]\n",
      "  Evaluated 2 individuals\n",
      "  Min 0.0697139352709\n",
      "  Max 0.0776996060331\n",
      "  Avg 0.0725883944471\n",
      "  Std 0.00268531884687\n",
      "-- Generation 6 --\n",
      "mate\n",
      "mate\n",
      "\n",
      "        l1_out:\t128\n",
      "        l2_out:\t128\n",
      "        l1_drop:\t0.023943619120116687\n",
      "        l2_drop:\t0.12502013021715647\n",
      "        bn1:\t0\n",
      "        bn2:\t0\n",
      "        batch_size:\t10\n",
      "        epochs:\t10\n",
      "        validation_split:\t0.20383943374115368\n",
      "        \n",
      "Epoch 00005: early stopping\n",
      "[0.074526480869813894, 0.97729999601840978]\n",
      "\n",
      "        l1_out:\t128\n",
      "        l2_out:\t128\n",
      "        l1_drop:\t0.023943619120116687\n",
      "        l2_drop:\t0.12502013021715647\n",
      "        bn1:\t0\n",
      "        bn2:\t0\n",
      "        batch_size:\t10\n",
      "        epochs:\t10\n",
      "        validation_split:\t0.20383943374115368\n",
      "        \n",
      "Epoch 00005: early stopping\n",
      "[0.073612029567364512, 0.97729999631643294]\n",
      "\n",
      "        l1_out:\t128\n",
      "        l2_out:\t128\n",
      "        l1_drop:\t0.023943619120116687\n",
      "        l2_drop:\t0.12502013021715647\n",
      "        bn1:\t0\n",
      "        bn2:\t0\n",
      "        batch_size:\t10\n",
      "        epochs:\t10\n",
      "        validation_split:\t0.20383943374115368\n",
      "        \n",
      "Epoch 00006: early stopping\n",
      "[0.072756685223521342, 0.97919999623298648]\n",
      "\n",
      "        l1_out:\t128\n",
      "        l2_out:\t128\n",
      "        l1_drop:\t0.023943619120116687\n",
      "        l2_drop:\t0.12502013021715647\n",
      "        bn1:\t0\n",
      "        bn2:\t0\n",
      "        batch_size:\t10\n",
      "        epochs:\t10\n",
      "        validation_split:\t0.20383943374115368\n",
      "        \n",
      "Epoch 00004: early stopping\n",
      "[0.078053687654439277, 0.97559999620914462]\n",
      "  Evaluated 4 individuals\n",
      "  Min 0.0697139352709\n",
      "  Max 0.0780536876544\n",
      "  Avg 0.0737325637172\n",
      "  Std 0.00269883895153\n",
      "-- Generation 7 --\n",
      "mutate\n",
      "mutate\n",
      "\n",
      "        l1_out:\t128\n",
      "        l2_out:\t128\n",
      "        l1_drop:\t0.023943619120116687\n",
      "        l2_drop:\t0.0\n",
      "        bn1:\t0\n",
      "        bn2:\t0\n",
      "        batch_size:\t10\n",
      "        epochs:\t10\n",
      "        validation_split:\t0.20383943374115368\n",
      "        \n",
      "Epoch 00004: early stopping\n",
      "[0.078657316071843522, 0.97699999642372126]\n",
      "\n",
      "        l1_out:\t128\n",
      "        l2_out:\t128\n",
      "        l1_drop:\t0.023943619120116687\n",
      "        l2_drop:\t0.12502013021715647\n",
      "        bn1:\t0\n",
      "        bn2:\t0\n",
      "        batch_size:\t10\n",
      "        epochs:\t10\n",
      "        validation_split:\t0.20383943374115368\n",
      "        \n",
      "Epoch 00005: early stopping\n",
      "[0.075296818633226389, 0.9768999963402748]\n",
      "  Evaluated 2 individuals\n",
      "  Min 0.0697139352709\n",
      "  Max 0.0786573160718\n",
      "  Avg 0.0726191881035\n",
      "  Std 0.00371349368332\n",
      "-- Generation 8 --\n",
      "  Evaluated 0 individuals\n",
      "  Min 0.0697139352709\n",
      "  Max 0.0697139352709\n",
      "  Avg 0.0697139352709\n",
      "  Std 0.0\n",
      "-- Generation 9 --\n",
      "mate\n",
      "mate\n",
      "mutate\n",
      "\n",
      "        l1_out:\t128\n",
      "        l2_out:\t128\n",
      "        l1_drop:\t0.023943619120116687\n",
      "        l2_drop:\t0.12502013021715647\n",
      "        bn1:\t0\n",
      "        bn2:\t0\n",
      "        batch_size:\t10\n",
      "        epochs:\t10\n",
      "        validation_split:\t0.20383943374115368\n",
      "        \n",
      "Epoch 00007: early stopping\n",
      "[0.07248481787607125, 0.97679999619722369]\n",
      "\n",
      "        l1_out:\t128\n",
      "        l2_out:\t128\n",
      "        l1_drop:\t0.023943619120116687\n",
      "        l2_drop:\t0.12502013021715647\n",
      "        bn1:\t0\n",
      "        bn2:\t0\n",
      "        batch_size:\t10\n",
      "        epochs:\t10\n",
      "        validation_split:\t0.20383943374115368\n",
      "        \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 00007: early stopping\n",
      "[0.07404292898667382, 0.97839999610185624]\n",
      "\n",
      "        l1_out:\t128\n",
      "        l2_out:\t128\n",
      "        l1_drop:\t0.023943619120116687\n",
      "        l2_drop:\t0.12502013021715647\n",
      "        bn1:\t0\n",
      "        bn2:\t0\n",
      "        batch_size:\t10\n",
      "        epochs:\t10\n",
      "        validation_split:\t0.20383943374115368\n",
      "        \n",
      "Epoch 00003: early stopping\n",
      "[0.089238391397235314, 0.9727999957799911]\n",
      "\n",
      "        l1_out:\t128\n",
      "        l2_out:\t128\n",
      "        l1_drop:\t0.023943619120116687\n",
      "        l2_drop:\t0.12502013021715647\n",
      "        bn1:\t0\n",
      "        bn2:\t0\n",
      "        batch_size:\t10\n",
      "        epochs:\t10\n",
      "        validation_split:\t0.20383943374115368\n",
      "        \n",
      "Epoch 00005: early stopping\n",
      "[0.071397722922849419, 0.97839999610185624]\n",
      "  Evaluated 4 individuals\n",
      "  Min 0.0697139352709\n",
      "  Max 0.0892383913972\n",
      "  Avg 0.0753755592907\n",
      "  Std 0.00707371998567\n",
      "-- End of (successful) evolution --\n",
      "Best individual is [128, 128, 0.023943619120116687, 0.12502013021715647, 0, 0, 10, 10, 0.20383943374115368], (0.069713935270881849,)\n"
     ]
    }
   ],
   "source": [
    "best_int = genAlg(population=5, CXPB=0.5, MUTPB=0.2, NGEN=10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### The output"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[128, 128, 0.023943619120116687, 0.12502013021715647, 0, 0, 10, 10, 0.20383943374115368]\n"
     ]
    }
   ],
   "source": [
    "print(best_int)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "kernelspec": {
   "display_name": "Python [default]",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}