Hand_Gesture_Recognition-DL_Project.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Gesture Recognition-->\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "## !pip install --upgrade scipy==1.1.0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "import datetime\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import os\n",
    "import cv2\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "from scipy.misc import imread, imresize\n",
    "import datetime \n",
    "from PIL import Image\n",
    "\n",
    "import warnings\n",
    "warnings.filterwarnings('ignore')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "from keras.callbacks import ModelCheckpoint, ReduceLROnPlateau\n",
    "from keras import optimizers\n",
    "from keras.losses import categorical_crossentropy\n",
    "from keras.optimizers import Adam,SGD\n",
    "from keras.models import Sequential, Model\n",
    "from keras.layers import Dense, GRU, Flatten, TimeDistributed, Dropout, BatchNormalization, ZeroPadding3D, Activation\n",
    "from keras.layers.convolutional import Conv3D, MaxPooling3D, Conv2D, MaxPooling2D\n",
    "from keras.layers.recurrent import LSTM, GRU"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We set the random seed so that the results don't vary drastically."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "np.random.seed(30)\n",
    "import random as rn\n",
    "rn.seed(30)\n",
    "from keras import backend as K\n",
    "import tensorflow as tf\n",
    "tf.set_random_seed(30) "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "In this block, you read the folder names for training and validation. You also set the `batch_size` here. Note that you set the batch size in such a way that you are able to use the GPU in full capacity. You keep increasing the batch size until the machine throws an error."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "train_doc = np.random.permutation(open('./Project_data/train.csv').readlines())\n",
    "val_doc = np.random.permutation(open('./Project_data/val.csv').readlines())\n",
    "\n",
    "batch_size = 32 #experiment with the batch size"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Generator\n",
    "This is one of the most important part of the code. The overall structure of the generator has been given. In the generator, you are going to preprocess the images as you have images of 2 different dimensions as well as create a batch of video frames. You have to experiment with `img_idx`, `y`,`z` and normalization such that you get high accuracy."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "def generator(source_path, folder_list, batch_size):\n",
    "    print( 'Source path = ', source_path, '; batch size =', batch_size)\n",
    "    img_idx = [i for i in range(0,30,2)] #create a list of image numbers you want to use for a particular video\n",
    "    # considering all the even indices from 0 to 28 with a step of 2 (odd frames from frame 1 to 29 within the video)\n",
    "    \n",
    "    x= len(img_idx) #x: number of images used for each video\n",
    "    y= 80\n",
    "    z= 80\n",
    "    # (y,z) is the final size of the input images and 3 is the number of channels RGB\n",
    "    #Assuming the size of cropped input image is consistent: 80x80\n",
    "    \n",
    "    while True:\n",
    "        t = np.random.permutation(folder_list)\n",
    "        num_batches = len(t)//(batch_size)  # calculate the number of batches\n",
    "        for batch in range(num_batches): # we iterate over the number of batches\n",
    "            batch_data = np.zeros((batch_size,x,y,z,3)) # x is the number of images you use for each video, (y,z) is the final size of the input images and 3 is the number of channels RGB\n",
    "            batch_labels = np.zeros((batch_size,5)) # batch_labels is the one hot representation of the output (output: 0,1,2,3,4. Hence 5 possible outputs)\n",
    "            for folder in range(batch_size): # iterate over the batch_size\n",
    "                imgs = os.listdir(source_path+'/'+ t[folder + (batch*batch_size)].split(';')[0]) # read all the images in the folder\n",
    "                for idx,item in enumerate(img_idx): #  Iterate over the frames/images of a folder to read them in\n",
    "                    image_path= source_path+'/'+ t[folder + (batch*batch_size)].strip().split(';')[0]+'/'+imgs[item]\n",
    "                    image = cv2.imread(image_path, cv2.IMREAD_COLOR).astype(np.float32)\n",
    "                    image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)\n",
    "                    \n",
    "                    #Crop the images and resize them. Note that the images are of 2 different shapes \n",
    "                    #and the conv3D will throw error if the inputs in a batch have different shapes\n",
    "                    \n",
    "                    if image.shape[0]!= image.shape[1]:  ##Checking for square frames 360x360\n",
    "                        image= image[0:120, 20:140, :]   ##Crops Rectangular frames of 120x160 to 120x120\n",
    "                    \n",
    "                    image = cv2.resize(image, (y,z), interpolation = cv2.INTER_AREA)  #Resizing 360x360 and 120x120 (cropped) images to 80x80\n",
    "\n",
    "                    batch_data[folder,idx,:,:,0] = (image[:,:,0])/255 #normalise and feed in the image\n",
    "                    batch_data[folder,idx,:,:,1] = (image[:,:,1])/255 #normalise and feed in the image\n",
    "                    batch_data[folder,idx,:,:,2] = (image[:,:,2])/255 #normalise and feed in the image\n",
    "                    \n",
    "                batch_labels[folder, int(t[folder + (batch*batch_size)].strip().split(';')[2])] = 1\n",
    "            yield batch_data, batch_labels #you yield the batch_data and the batch_labels, remember what does yield do\n",
    "\n",
    "        \n",
    "        # write the code for the remaining data points which are left after full batches\n",
    "        if len(t)!= num_batches*batch_size:\n",
    "            batch_size= len(t) - (num_batches*batch_size)\n",
    "            batch= num_batches\n",
    "            batch_data = np.zeros((batch_size,x,y,z,3)) # x is the number of images you use for each video, (y,z) is the final size of the input images and 3 is the number of channels RGB\n",
    "            batch_labels = np.zeros((batch_size,5)) # batch_labels is the one hot representation of the output (output: 0,1,2,3,4. Hence 5 possible outputs)\n",
    "            for folder in range(batch_size): # iterate over the batch_size\n",
    "                imgs = os.listdir(source_path+'/'+ t[folder + (batch*batch_size)].split(';')[0]) # read all the images in the folder\n",
    "                for idx,item in enumerate(img_idx): #  Iterate over the frames/images of a folder to read them in\n",
    "                    image_path= source_path+'/'+ t[folder + (batch*batch_size)].strip().split(';')[0]+'/'+imgs[item]\n",
    "                    image = cv2.imread(image_path, cv2.IMREAD_COLOR).astype(np.float32)\n",
    "                    image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)\n",
    "\n",
    "                    #crop the images and resize them. Note that the images are of 2 different shapes \n",
    "                    #and the conv3D will throw error if the inputs in a batch have different shapes\n",
    "\n",
    "                    if image.shape[0]!= image.shape[1]:  ##Checking for square frames 360x360\n",
    "                        image= image[0:120, 20:140, :]   ##Crops Rectangular frames to 120x120\n",
    "\n",
    "                    image = cv2.resize(image, (y,z), interpolation = cv2.INTER_AREA)  #Resizing 360x360 and 120x120 (cropped) images to 80x80\n",
    "\n",
    "                    batch_data[folder,idx,:,:,0] = (image[:,:,0])/255 #normalise and feed in the image\n",
    "                    batch_data[folder,idx,:,:,1] = (image[:,:,1])/255 #normalise and feed in the image\n",
    "                    batch_data[folder,idx,:,:,2] = (image[:,:,2])/255 #normalise and feed in the image\n",
    "    \n",
    "                batch_labels[folder, int(t[folder + (batch*batch_size)].strip().split(';')[2])] = 1\n",
    "            yield batch_data, batch_labels #you yield the batch_data and the batch_labels, remember what does yield do\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Note here that a video is represented above in the generator as (number of images, height, width, number of channels). Take this into consideration while creating the model architecture."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "# training sequences = 663\n",
      "# validation sequences = 100\n",
      "# epochs = 30\n"
     ]
    }
   ],
   "source": [
    "curr_dt_time = datetime.datetime.now()\n",
    "train_path = './Project_data/train'\n",
    "val_path = './Project_data/val'\n",
    "num_train_sequences = len(train_doc)\n",
    "print('# training sequences =', num_train_sequences)\n",
    "num_val_sequences = len(val_doc)\n",
    "print('# validation sequences =', num_val_sequences)\n",
    "num_epochs = 30 # choose the number of epochs\n",
    "print ('# epochs =', num_epochs)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Model\n",
    "Here you make the model using different functionalities that Keras provides. Remember to use `Conv3D` and `MaxPooling3D` and not `Conv2D` and `Maxpooling2D`. Also remember that the last layer is the softmax. Remember that the network is designed in such a way that the model is able to fit in the memory of the webcam."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Conv3D Models"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "input_shape= (15,80,80,3)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Model 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "## Model1\n",
    "\n",
    "# 16,16,32 kernels in 3 consecutive conv3D layers\n",
    "model1 = Sequential()\n",
    "# 1st Layer Group\n",
    "model1.add(Conv3D(16, kernel_size=(3,3,3), input_shape=input_shape,padding='same'))\n",
    "model1.add(BatchNormalization())\n",
    "model1.add(Activation('relu'))\n",
    "model1.add(MaxPooling3D(pool_size=(2,2,2)))\n",
    "\n",
    "\n",
    "# 2nd Layer Group\n",
    "model1.add(Conv3D(16, kernel_size=(3,3,3),padding='same'))\n",
    "model1.add(BatchNormalization())\n",
    "model1.add(Activation('relu'))\n",
    "model1.add(MaxPooling3D(pool_size=(2,2,2)))\n",
    "\n",
    "# 3rd Layer Group\n",
    "model1.add(Conv3D(32, kernel_size=(3,3,3),padding='same'))\n",
    "model1.add(BatchNormalization())\n",
    "model1.add(Activation('relu'))\n",
    "model1.add(MaxPooling3D(pool_size=(2,2,2)))\n",
    "\n",
    "# FC layer group\n",
    "model1.add(Flatten())\n",
    "model1.add(Dense(512 , activation='relu'))\n",
    "model1.add(Dropout(0.5))\n",
    "\n",
    "# Softmax Layer\n",
    "model1.add(Dense(5, activation='softmax'))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Now that you have written the model, the next step is to `compile` the model. When you print the `summary` of the model, you'll see the total number of parameters you have to train."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "conv3d_1 (Conv3D)            (None, 15, 80, 80, 16)    1312      \n",
      "_________________________________________________________________\n",
      "batch_normalization_1 (Batch (None, 15, 80, 80, 16)    64        \n",
      "_________________________________________________________________\n",
      "activation_1 (Activation)    (None, 15, 80, 80, 16)    0         \n",
      "_________________________________________________________________\n",
      "max_pooling3d_1 (MaxPooling3 (None, 7, 40, 40, 16)     0         \n",
      "_________________________________________________________________\n",
      "conv3d_2 (Conv3D)            (None, 7, 40, 40, 16)     6928      \n",
      "_________________________________________________________________\n",
      "batch_normalization_2 (Batch (None, 7, 40, 40, 16)     64        \n",
      "_________________________________________________________________\n",
      "activation_2 (Activation)    (None, 7, 40, 40, 16)     0         \n",
      "_________________________________________________________________\n",
      "max_pooling3d_2 (MaxPooling3 (None, 3, 20, 20, 16)     0         \n",
      "_________________________________________________________________\n",
      "conv3d_3 (Conv3D)            (None, 3, 20, 20, 32)     13856     \n",
      "_________________________________________________________________\n",
      "batch_normalization_3 (Batch (None, 3, 20, 20, 32)     128       \n",
      "_________________________________________________________________\n",
      "activation_3 (Activation)    (None, 3, 20, 20, 32)     0         \n",
      "_________________________________________________________________\n",
      "max_pooling3d_3 (MaxPooling3 (None, 1, 10, 10, 32)     0         \n",
      "_________________________________________________________________\n",
      "flatten_1 (Flatten)          (None, 3200)              0         \n",
      "_________________________________________________________________\n",
      "dense_1 (Dense)              (None, 512)               1638912   \n",
      "_________________________________________________________________\n",
      "dropout_1 (Dropout)          (None, 512)               0         \n",
      "_________________________________________________________________\n",
      "dense_2 (Dense)              (None, 5)                 2565      \n",
      "=================================================================\n",
      "Total params: 1,663,829\n",
      "Trainable params: 1,663,701\n",
      "Non-trainable params: 128\n",
      "_________________________________________________________________\n",
      "None\n"
     ]
    }
   ],
   "source": [
    "optimiser = Adam() #write your optimizer\n",
    "model1.compile(optimizer=optimiser, loss='categorical_crossentropy', metrics=['categorical_accuracy'])\n",
    "print (model1.summary())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Model 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Model 2\n",
    "# 16,32,32 kernels in 3 consecutive conv3D layers\n",
    "model2 = Sequential()\n",
    "# 1st Layer Group\n",
    "model2.add(Conv3D(16, kernel_size=(3,3,3), input_shape=input_shape,padding='same'))\n",
    "model2.add(BatchNormalization())\n",
    "model2.add(Activation('relu'))\n",
    "model2.add(MaxPooling3D(pool_size=(2,2,2)))\n",
    "\n",
    "\n",
    "# 2nd Layer Group\n",
    "model2.add(Conv3D(32, kernel_size=(3,3,3),padding='same'))\n",
    "model2.add(BatchNormalization())\n",
    "model2.add(Activation('relu'))\n",
    "model2.add(MaxPooling3D(pool_size=(2,2,2)))\n",
    "\n",
    "# 3rd Layer Group\n",
    "model2.add(Conv3D(32, kernel_size=(3,3,3),padding='same'))\n",
    "model2.add(BatchNormalization())\n",
    "model2.add(Activation('relu'))\n",
    "model2.add(MaxPooling3D(pool_size=(2,2,2)))\n",
    "\n",
    "# FC layer group\n",
    "model2.add(Flatten())\n",
    "model2.add(Dense(512, activation='relu'))\n",
    "model2.add(Dropout(0.5))\n",
    "# Softmax Layer\n",
    "model2.add(Dense(5, activation='softmax'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "conv3d_4 (Conv3D)            (None, 15, 80, 80, 16)    1312      \n",
      "_________________________________________________________________\n",
      "batch_normalization_4 (Batch (None, 15, 80, 80, 16)    64        \n",
      "_________________________________________________________________\n",
      "activation_4 (Activation)    (None, 15, 80, 80, 16)    0         \n",
      "_________________________________________________________________\n",
      "max_pooling3d_4 (MaxPooling3 (None, 7, 40, 40, 16)     0         \n",
      "_________________________________________________________________\n",
      "conv3d_5 (Conv3D)            (None, 7, 40, 40, 32)     13856     \n",
      "_________________________________________________________________\n",
      "batch_normalization_5 (Batch (None, 7, 40, 40, 32)     128       \n",
      "_________________________________________________________________\n",
      "activation_5 (Activation)    (None, 7, 40, 40, 32)     0         \n",
      "_________________________________________________________________\n",
      "max_pooling3d_5 (MaxPooling3 (None, 3, 20, 20, 32)     0         \n",
      "_________________________________________________________________\n",
      "conv3d_6 (Conv3D)            (None, 3, 20, 20, 32)     27680     \n",
      "_________________________________________________________________\n",
      "batch_normalization_6 (Batch (None, 3, 20, 20, 32)     128       \n",
      "_________________________________________________________________\n",
      "activation_6 (Activation)    (None, 3, 20, 20, 32)     0         \n",
      "_________________________________________________________________\n",
      "max_pooling3d_6 (MaxPooling3 (None, 1, 10, 10, 32)     0         \n",
      "_________________________________________________________________\n",
      "flatten_2 (Flatten)          (None, 3200)              0         \n",
      "_________________________________________________________________\n",
      "dense_3 (Dense)              (None, 512)               1638912   \n",
      "_________________________________________________________________\n",
      "dropout_2 (Dropout)          (None, 512)               0         \n",
      "_________________________________________________________________\n",
      "dense_4 (Dense)              (None, 5)                 2565      \n",
      "=================================================================\n",
      "Total params: 1,684,645\n",
      "Trainable params: 1,684,485\n",
      "Non-trainable params: 160\n",
      "_________________________________________________________________\n",
      "None\n"
     ]
    }
   ],
   "source": [
    "optimiser = Adam() #write your optimizer\n",
    "model2.compile(optimizer=optimiser, loss='categorical_crossentropy', metrics=['categorical_accuracy'])\n",
    "print (model2.summary())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Model 3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Model 3\n",
    "# 16,32,64 kernels in 3 consecutive conv3D layers\n",
    "model3 = Sequential()\n",
    "# 1st Layer Group\n",
    "model3.add(Conv3D(16, kernel_size=(3,3,3), input_shape=input_shape,padding='same'))\n",
    "model3.add(BatchNormalization())\n",
    "model3.add(Activation('relu'))\n",
    "model3.add(MaxPooling3D(pool_size=(2,2,2)))\n",
    "\n",
    "\n",
    "# 2nd Layer Group\n",
    "model3.add(Conv3D(32, kernel_size=(3,3,3),padding='same'))\n",
    "model3.add(BatchNormalization())\n",
    "model3.add(Activation('relu'))\n",
    "model3.add(MaxPooling3D(pool_size=(2,2,2)))\n",
    "\n",
    "# 3rd Layer Group\n",
    "model3.add(Conv3D(64, kernel_size=(3,3,3),padding='same'))\n",
    "model3.add(BatchNormalization())\n",
    "model3.add(Activation('relu'))\n",
    "model3.add(MaxPooling3D(pool_size=(2,2,2)))\n",
    "\n",
    "# FC layer group\n",
    "model3.add(Flatten())\n",
    "model3.add(Dense(512, activation='relu'))\n",
    "model3.add(Dropout(0.5))\n",
    "# Softmax Layer\n",
    "model3.add(Dense(5, activation='softmax'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "conv3d_7 (Conv3D)            (None, 15, 80, 80, 16)    1312      \n",
      "_________________________________________________________________\n",
      "batch_normalization_7 (Batch (None, 15, 80, 80, 16)    64        \n",
      "_________________________________________________________________\n",
      "activation_7 (Activation)    (None, 15, 80, 80, 16)    0         \n",
      "_________________________________________________________________\n",
      "max_pooling3d_7 (MaxPooling3 (None, 7, 40, 40, 16)     0         \n",
      "_________________________________________________________________\n",
      "conv3d_8 (Conv3D)            (None, 7, 40, 40, 32)     13856     \n",
      "_________________________________________________________________\n",
      "batch_normalization_8 (Batch (None, 7, 40, 40, 32)     128       \n",
      "_________________________________________________________________\n",
      "activation_8 (Activation)    (None, 7, 40, 40, 32)     0         \n",
      "_________________________________________________________________\n",
      "max_pooling3d_8 (MaxPooling3 (None, 3, 20, 20, 32)     0         \n",
      "_________________________________________________________________\n",
      "conv3d_9 (Conv3D)            (None, 3, 20, 20, 64)     55360     \n",
      "_________________________________________________________________\n",
      "batch_normalization_9 (Batch (None, 3, 20, 20, 64)     256       \n",
      "_________________________________________________________________\n",
      "activation_9 (Activation)    (None, 3, 20, 20, 64)     0         \n",
      "_________________________________________________________________\n",
      "max_pooling3d_9 (MaxPooling3 (None, 1, 10, 10, 64)     0         \n",
      "_________________________________________________________________\n",
      "flatten_3 (Flatten)          (None, 6400)              0         \n",
      "_________________________________________________________________\n",
      "dense_5 (Dense)              (None, 512)               3277312   \n",
      "_________________________________________________________________\n",
      "dropout_3 (Dropout)          (None, 512)               0         \n",
      "_________________________________________________________________\n",
      "dense_6 (Dense)              (None, 5)                 2565      \n",
      "=================================================================\n",
      "Total params: 3,350,853\n",
      "Trainable params: 3,350,629\n",
      "Non-trainable params: 224\n",
      "_________________________________________________________________\n",
      "None\n"
     ]
    }
   ],
   "source": [
    "optimiser = Adam() #write your optimizer\n",
    "model3.compile(optimizer=optimiser, loss='categorical_crossentropy', metrics=['categorical_accuracy'])\n",
    "print (model3.summary())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Model 4"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Model 4\n",
    "# 16,64,64 kernels in 3 consecutive conv3D layers\n",
    "model4 = Sequential()\n",
    "# 1st Layer Group\n",
    "model4.add(Conv3D(16, kernel_size=(3,3,3), input_shape=input_shape,padding='same'))\n",
    "model4.add(BatchNormalization())\n",
    "model4.add(Activation('relu'))\n",
    "model4.add(MaxPooling3D(pool_size=(2,2,2)))\n",
    "\n",
    "\n",
    "# 2nd Layer Group\n",
    "model4.add(Conv3D(64, kernel_size=(3,3,3),padding='same'))\n",
    "model4.add(BatchNormalization())\n",
    "model4.add(Activation('relu'))\n",
    "model4.add(MaxPooling3D(pool_size=(2,2,2)))\n",
    "\n",
    "# 3rd Layer Group\n",
    "model4.add(Conv3D(64, kernel_size=(3,3,3),padding='same'))\n",
    "model4.add(BatchNormalization())\n",
    "model4.add(Activation('relu'))\n",
    "model4.add(MaxPooling3D(pool_size=(2,2,2)))\n",
    "\n",
    "# FC layer group\n",
    "model4.add(Flatten())\n",
    "model4.add(Dense(512, activation='relu'))\n",
    "model4.add(Dropout(0.5))\n",
    "# Softmax Layer\n",
    "model4.add(Dense(5, activation='softmax'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "conv3d_10 (Conv3D)           (None, 15, 80, 80, 16)    1312      \n",
      "_________________________________________________________________\n",
      "batch_normalization_10 (Batc (None, 15, 80, 80, 16)    64        \n",
      "_________________________________________________________________\n",
      "activation_10 (Activation)   (None, 15, 80, 80, 16)    0         \n",
      "_________________________________________________________________\n",
      "max_pooling3d_10 (MaxPooling (None, 7, 40, 40, 16)     0         \n",
      "_________________________________________________________________\n",
      "conv3d_11 (Conv3D)           (None, 7, 40, 40, 64)     27712     \n",
      "_________________________________________________________________\n",
      "batch_normalization_11 (Batc (None, 7, 40, 40, 64)     256       \n",
      "_________________________________________________________________\n",
      "activation_11 (Activation)   (None, 7, 40, 40, 64)     0         \n",
      "_________________________________________________________________\n",
      "max_pooling3d_11 (MaxPooling (None, 3, 20, 20, 64)     0         \n",
      "_________________________________________________________________\n",
      "conv3d_12 (Conv3D)           (None, 3, 20, 20, 64)     110656    \n",
      "_________________________________________________________________\n",
      "batch_normalization_12 (Batc (None, 3, 20, 20, 64)     256       \n",
      "_________________________________________________________________\n",
      "activation_12 (Activation)   (None, 3, 20, 20, 64)     0         \n",
      "_________________________________________________________________\n",
      "max_pooling3d_12 (MaxPooling (None, 1, 10, 10, 64)     0         \n",
      "_________________________________________________________________\n",
      "flatten_4 (Flatten)          (None, 6400)              0         \n",
      "_________________________________________________________________\n",
      "dense_7 (Dense)              (None, 512)               3277312   \n",
      "_________________________________________________________________\n",
      "dropout_4 (Dropout)          (None, 512)               0         \n",
      "_________________________________________________________________\n",
      "dense_8 (Dense)              (None, 5)                 2565      \n",
      "=================================================================\n",
      "Total params: 3,420,133\n",
      "Trainable params: 3,419,845\n",
      "Non-trainable params: 288\n",
      "_________________________________________________________________\n",
      "None\n"
     ]
    }
   ],
   "source": [
    "optimiser = Adam() #write your optimizer\n",
    "model4.compile(optimizer=optimiser, loss='categorical_crossentropy', metrics=['categorical_accuracy'])\n",
    "print (model4.summary())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Let us create the `train_generator` and the `val_generator` which will be used in `.fit_generator`."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Model 5"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Model 5\n",
    "# 16,32,32 kernels in 3 consecutive conv3D layers\n",
    "model5 = Sequential()\n",
    "# 1st Layer Group\n",
    "model5.add(Conv3D(16, kernel_size=(2,2,2), input_shape=input_shape,padding='same'))\n",
    "model5.add(BatchNormalization())\n",
    "model5.add(Activation('relu'))\n",
    "model5.add(MaxPooling3D(pool_size=(2,2,2)))\n",
    "\n",
    "\n",
    "# 2nd Layer Group\n",
    "model5.add(Conv3D(32, kernel_size=(2,2,2),padding='same'))\n",
    "model5.add(BatchNormalization())\n",
    "model5.add(Activation('relu'))\n",
    "model5.add(MaxPooling3D(pool_size=(2,2,2)))\n",
    "\n",
    "# 3rd Layer Group\n",
    "model5.add(Conv3D(32, kernel_size=(2,2,2),padding='same'))\n",
    "model5.add(BatchNormalization())\n",
    "model5.add(Activation('relu'))\n",
    "model5.add(MaxPooling3D(pool_size=(2,2,2)))\n",
    "\n",
    "# FC layer group\n",
    "model5.add(Flatten())\n",
    "model5.add(Dense(512,activation='relu'))\n",
    "model5.add(Dropout(0.5))\n",
    "\n",
    "model5.add(Dense(128,activation='relu'))\n",
    "model5.add(Dropout(0.25))\n",
    "\n",
    "# Softmax Layer\n",
    "model5.add(Dense(5, activation='softmax'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "conv3d_13 (Conv3D)           (None, 15, 80, 80, 16)    400       \n",
      "_________________________________________________________________\n",
      "batch_normalization_13 (Batc (None, 15, 80, 80, 16)    64        \n",
      "_________________________________________________________________\n",
      "activation_13 (Activation)   (None, 15, 80, 80, 16)    0         \n",
      "_________________________________________________________________\n",
      "max_pooling3d_13 (MaxPooling (None, 7, 40, 40, 16)     0         \n",
      "_________________________________________________________________\n",
      "conv3d_14 (Conv3D)           (None, 7, 40, 40, 32)     4128      \n",
      "_________________________________________________________________\n",
      "batch_normalization_14 (Batc (None, 7, 40, 40, 32)     128       \n",
      "_________________________________________________________________\n",
      "activation_14 (Activation)   (None, 7, 40, 40, 32)     0         \n",
      "_________________________________________________________________\n",
      "max_pooling3d_14 (MaxPooling (None, 3, 20, 20, 32)     0         \n",
      "_________________________________________________________________\n",
      "conv3d_15 (Conv3D)           (None, 3, 20, 20, 32)     8224      \n",
      "_________________________________________________________________\n",
      "batch_normalization_15 (Batc (None, 3, 20, 20, 32)     128       \n",
      "_________________________________________________________________\n",
      "activation_15 (Activation)   (None, 3, 20, 20, 32)     0         \n",
      "_________________________________________________________________\n",
      "max_pooling3d_15 (MaxPooling (None, 1, 10, 10, 32)     0         \n",
      "_________________________________________________________________\n",
      "flatten_5 (Flatten)          (None, 3200)              0         \n",
      "_________________________________________________________________\n",
      "dense_9 (Dense)              (None, 512)               1638912   \n",
      "_________________________________________________________________\n",
      "dropout_5 (Dropout)          (None, 512)               0         \n",
      "_________________________________________________________________\n",
      "dense_10 (Dense)             (None, 128)               65664     \n",
      "_________________________________________________________________\n",
      "dropout_6 (Dropout)          (None, 128)               0         \n",
      "_________________________________________________________________\n",
      "dense_11 (Dense)             (None, 5)                 645       \n",
      "=================================================================\n",
      "Total params: 1,718,293\n",
      "Trainable params: 1,718,133\n",
      "Non-trainable params: 160\n",
      "_________________________________________________________________\n",
      "None\n"
     ]
    }
   ],
   "source": [
    "optimiser = Adam() #write your optimizer\n",
    "model5.compile(optimizer=optimiser, loss='categorical_crossentropy', metrics=['categorical_accuracy'])\n",
    "print (model5.summary())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Model 6"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Model 6\n",
    "# 16,32,32 kernels in 3 consecutive conv3D layers\n",
    "model6 = Sequential()\n",
    "# 1st Layer Group\n",
    "model6.add(Conv3D(16, kernel_size=(5,5,5), input_shape=input_shape,padding='same'))\n",
    "model6.add(BatchNormalization())\n",
    "model6.add(Activation('relu'))\n",
    "model6.add(MaxPooling3D(pool_size=(2,2,2)))\n",
    "\n",
    "\n",
    "# 2nd Layer Group\n",
    "model6.add(Conv3D(32, kernel_size=(5,5,5),padding='same'))\n",
    "model6.add(BatchNormalization())\n",
    "model6.add(Activation('relu'))\n",
    "model6.add(MaxPooling3D(pool_size=(2,2,2)))\n",
    "\n",
    "# 3rd Layer Group\n",
    "model6.add(Conv3D(32, kernel_size=(5,5,5),padding='same'))\n",
    "model6.add(BatchNormalization())\n",
    "model6.add(Activation('relu'))\n",
    "model6.add(MaxPooling3D(pool_size=(2,2,2)))\n",
    "\n",
    "#FC layer group\n",
    "model6.add(Flatten())\n",
    "model6.add(Dense(512,activation='relu'))\n",
    "model6.add(Dropout(0.5))\n",
    "\n",
    "model6.add(Dense(128,activation='relu'))\n",
    "model6.add(Dropout(0.25))\n",
    "\n",
    "# Softmax Layer\n",
    "model6.add(Dense(5, activation='softmax'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "conv3d_16 (Conv3D)           (None, 15, 80, 80, 16)    6016      \n",
      "_________________________________________________________________\n",
      "batch_normalization_16 (Batc (None, 15, 80, 80, 16)    64        \n",
      "_________________________________________________________________\n",
      "activation_16 (Activation)   (None, 15, 80, 80, 16)    0         \n",
      "_________________________________________________________________\n",
      "max_pooling3d_16 (MaxPooling (None, 7, 40, 40, 16)     0         \n",
      "_________________________________________________________________\n",
      "conv3d_17 (Conv3D)           (None, 7, 40, 40, 32)     64032     \n",
      "_________________________________________________________________\n",
      "batch_normalization_17 (Batc (None, 7, 40, 40, 32)     128       \n",
      "_________________________________________________________________\n",
      "activation_17 (Activation)   (None, 7, 40, 40, 32)     0         \n",
      "_________________________________________________________________\n",
      "max_pooling3d_17 (MaxPooling (None, 3, 20, 20, 32)     0         \n",
      "_________________________________________________________________\n",
      "conv3d_18 (Conv3D)           (None, 3, 20, 20, 32)     128032    \n",
      "_________________________________________________________________\n",
      "batch_normalization_18 (Batc (None, 3, 20, 20, 32)     128       \n",
      "_________________________________________________________________\n",
      "activation_18 (Activation)   (None, 3, 20, 20, 32)     0         \n",
      "_________________________________________________________________\n",
      "max_pooling3d_18 (MaxPooling (None, 1, 10, 10, 32)     0         \n",
      "_________________________________________________________________\n",
      "flatten_6 (Flatten)          (None, 3200)              0         \n",
      "_________________________________________________________________\n",
      "dense_12 (Dense)             (None, 512)               1638912   \n",
      "_________________________________________________________________\n",
      "dropout_7 (Dropout)          (None, 512)               0         \n",
      "_________________________________________________________________\n",
      "dense_13 (Dense)             (None, 128)               65664     \n",
      "_________________________________________________________________\n",
      "dropout_8 (Dropout)          (None, 128)               0         \n",
      "_________________________________________________________________\n",
      "dense_14 (Dense)             (None, 5)                 645       \n",
      "=================================================================\n",
      "Total params: 1,903,621\n",
      "Trainable params: 1,903,461\n",
      "Non-trainable params: 160\n",
      "_________________________________________________________________\n",
      "None\n"
     ]
    }
   ],
   "source": [
    "optimiser = Adam() #write your optimizer\n",
    "model6.compile(optimizer=optimiser, loss='categorical_crossentropy', metrics=['categorical_accuracy'])\n",
    "print (model6.summary())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Model 7"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Model 7\n",
    "# 32,64,64 kernels in 3 consecutive conv3D layers\n",
    "model7 = Sequential()\n",
    "# 1st Layer Group\n",
    "model7.add(Conv3D(32, kernel_size=(3,3,3), strides=(1,1,1) , input_shape=input_shape,padding='same'))\n",
    "model7.add(BatchNormalization())\n",
    "model7.add(Activation('relu'))\n",
    "model7.add(MaxPooling3D(pool_size=(2,2,2)))\n",
    "\n",
    "\n",
    "# 2nd Layer Group\n",
    "model7.add(Conv3D(64, kernel_size=(3,3,3),strides=(1,1,1) ,padding='same'))\n",
    "model7.add(BatchNormalization())\n",
    "model7.add(Activation('relu'))\n",
    "model7.add(MaxPooling3D(pool_size=(2,2,2)))\n",
    "\n",
    "# 3rd Layer Group\n",
    "model7.add(Conv3D(64, kernel_size=(3,3,3),padding='same'))\n",
    "model7.add(BatchNormalization())\n",
    "model7.add(Activation('relu'))\n",
    "model7.add(MaxPooling3D(pool_size=(2,2,2)))\n",
    "\n",
    "\n",
    "#FC layer group\n",
    "model7.add(Flatten())\n",
    "model7.add(Dense(512,activation='relu'))\n",
    "model7.add(Dropout(0.5))\n",
    "\n",
    "model7.add(Dense(128,activation='relu'))\n",
    "model7.add(Dropout(0.25))\n",
    "\n",
    "# Softmax Layer\n",
    "model7.add(Dense(5, activation='softmax'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "conv3d_19 (Conv3D)           (None, 15, 80, 80, 32)    2624      \n",
      "_________________________________________________________________\n",
      "batch_normalization_19 (Batc (None, 15, 80, 80, 32)    128       \n",
      "_________________________________________________________________\n",
      "activation_19 (Activation)   (None, 15, 80, 80, 32)    0         \n",
      "_________________________________________________________________\n",
      "max_pooling3d_19 (MaxPooling (None, 7, 40, 40, 32)     0         \n",
      "_________________________________________________________________\n",
      "conv3d_20 (Conv3D)           (None, 7, 40, 40, 64)     55360     \n",
      "_________________________________________________________________\n",
      "batch_normalization_20 (Batc (None, 7, 40, 40, 64)     256       \n",
      "_________________________________________________________________\n",
      "activation_20 (Activation)   (None, 7, 40, 40, 64)     0         \n",
      "_________________________________________________________________\n",
      "max_pooling3d_20 (MaxPooling (None, 3, 20, 20, 64)     0         \n",
      "_________________________________________________________________\n",
      "conv3d_21 (Conv3D)           (None, 3, 20, 20, 64)     110656    \n",
      "_________________________________________________________________\n",
      "batch_normalization_21 (Batc (None, 3, 20, 20, 64)     256       \n",
      "_________________________________________________________________\n",
      "activation_21 (Activation)   (None, 3, 20, 20, 64)     0         \n",
      "_________________________________________________________________\n",
      "max_pooling3d_21 (MaxPooling (None, 1, 10, 10, 64)     0         \n",
      "_________________________________________________________________\n",
      "flatten_7 (Flatten)          (None, 6400)              0         \n",
      "_________________________________________________________________\n",
      "dense_15 (Dense)             (None, 512)               3277312   \n",
      "_________________________________________________________________\n",
      "dropout_9 (Dropout)          (None, 512)               0         \n",
      "_________________________________________________________________\n",
      "dense_16 (Dense)             (None, 128)               65664     \n",
      "_________________________________________________________________\n",
      "dropout_10 (Dropout)         (None, 128)               0         \n",
      "_________________________________________________________________\n",
      "dense_17 (Dense)             (None, 5)                 645       \n",
      "=================================================================\n",
      "Total params: 3,512,901\n",
      "Trainable params: 3,512,581\n",
      "Non-trainable params: 320\n",
      "_________________________________________________________________\n",
      "None\n"
     ]
    }
   ],
   "source": [
    "optimiser = Adam() #write your optimizer\n",
    "model7.compile(optimizer=optimiser, loss='categorical_crossentropy', metrics=['categorical_accuracy'])\n",
    "print (model7.summary())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Conv2D+RNN (LSTM) Models"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    " ## Model 8"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [],
   "source": [
    "Input_shape = (15,80,80,3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Model8\n",
    "model8 = Sequential()\n",
    "\n",
    "#1st Conv2D group\n",
    "model8.add(TimeDistributed(Conv2D(32, (3, 3) , padding='same', activation='relu'), input_shape=Input_shape))\n",
    "model8.add(TimeDistributed(BatchNormalization()))\n",
    "model8.add(TimeDistributed(Conv2D(32, (3, 3) , padding='same', activation='relu')))\n",
    "model8.add(TimeDistributed(BatchNormalization()))\n",
    "model8.add(TimeDistributed(MaxPooling2D((2, 2))))\n",
    "\n",
    "#2nd Conv2D group\n",
    "model8.add(TimeDistributed(Conv2D(64, (3, 3) , padding='same', activation='relu'), input_shape=Input_shape))\n",
    "model8.add(TimeDistributed(BatchNormalization()))\n",
    "model8.add(TimeDistributed(Conv2D(64, (3, 3) , padding='same', activation='relu')))\n",
    "model8.add(TimeDistributed(BatchNormalization()))\n",
    "model8.add(TimeDistributed(MaxPooling2D((2, 2))))\n",
    "\n",
    "#3rd Conv2D group\n",
    "model8.add(TimeDistributed(Conv2D(128, (3, 3) , padding='same', activation='relu'), input_shape=Input_shape))\n",
    "model8.add(TimeDistributed(BatchNormalization()))\n",
    "model8.add(TimeDistributed(Conv2D(128, (3, 3) , padding='same', activation='relu')))\n",
    "model8.add(TimeDistributed(BatchNormalization()))\n",
    "model8.add(TimeDistributed(MaxPooling2D((2, 2))))\n",
    "\n",
    "#FC layer group\n",
    "model8.add(TimeDistributed(Flatten()))\n",
    "#LSTM\n",
    "model8.add(LSTM(128, return_sequences=False, dropout=0.5))\n",
    "model8.add(Dense(64,activation='relu'))\n",
    "model8.add(Dropout(0.5))\n",
    "\n",
    "#Softmax layer\n",
    "model8.add(Dense(5, activation='softmax'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "time_distributed_1 (TimeDist (None, 15, 80, 80, 32)    896       \n",
      "_________________________________________________________________\n",
      "time_distributed_2 (TimeDist (None, 15, 80, 80, 32)    128       \n",
      "_________________________________________________________________\n",
      "time_distributed_3 (TimeDist (None, 15, 80, 80, 32)    9248      \n",
      "_________________________________________________________________\n",
      "time_distributed_4 (TimeDist (None, 15, 80, 80, 32)    128       \n",
      "_________________________________________________________________\n",
      "time_distributed_5 (TimeDist (None, 15, 40, 40, 32)    0         \n",
      "_________________________________________________________________\n",
      "time_distributed_6 (TimeDist (None, 15, 40, 40, 64)    18496     \n",
      "_________________________________________________________________\n",
      "time_distributed_7 (TimeDist (None, 15, 40, 40, 64)    256       \n",
      "_________________________________________________________________\n",
      "time_distributed_8 (TimeDist (None, 15, 40, 40, 64)    36928     \n",
      "_________________________________________________________________\n",
      "time_distributed_9 (TimeDist (None, 15, 40, 40, 64)    256       \n",
      "_________________________________________________________________\n",
      "time_distributed_10 (TimeDis (None, 15, 20, 20, 64)    0         \n",
      "_________________________________________________________________\n",
      "time_distributed_11 (TimeDis (None, 15, 20, 20, 128)   73856     \n",
      "_________________________________________________________________\n",
      "time_distributed_12 (TimeDis (None, 15, 20, 20, 128)   512       \n",
      "_________________________________________________________________\n",
      "time_distributed_13 (TimeDis (None, 15, 20, 20, 128)   147584    \n",
      "_________________________________________________________________\n",
      "time_distributed_14 (TimeDis (None, 15, 20, 20, 128)   512       \n",
      "_________________________________________________________________\n",
      "time_distributed_15 (TimeDis (None, 15, 10, 10, 128)   0         \n",
      "_________________________________________________________________\n",
      "time_distributed_16 (TimeDis (None, 15, 12800)         0         \n",
      "_________________________________________________________________\n",
      "lstm_1 (LSTM)                (None, 128)               6619648   \n",
      "_________________________________________________________________\n",
      "dense_18 (Dense)             (None, 64)                8256      \n",
      "_________________________________________________________________\n",
      "dropout_11 (Dropout)         (None, 64)                0         \n",
      "_________________________________________________________________\n",
      "dense_19 (Dense)             (None, 5)                 325       \n",
      "=================================================================\n",
      "Total params: 6,917,029\n",
      "Trainable params: 6,916,133\n",
      "Non-trainable params: 896\n",
      "_________________________________________________________________\n",
      "None\n"
     ]
    }
   ],
   "source": [
    "optimiser = Adam() #write your optimizer\n",
    "model8.compile(optimizer=optimiser, loss='categorical_crossentropy', metrics=['categorical_accuracy'])\n",
    "print (model8.summary())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    " ## Model 9"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Model9\n",
    "model9 = Sequential()\n",
    "\n",
    "#1st Conv2D group\n",
    "model9.add(TimeDistributed(Conv2D(16, (3, 3) , padding='same', activation='relu'), input_shape=Input_shape))\n",
    "model9.add(TimeDistributed(BatchNormalization()))\n",
    "model9.add(TimeDistributed(Conv2D(16, (3, 3) , padding='same', activation='relu')))\n",
    "model9.add(TimeDistributed(BatchNormalization()))\n",
    "model9.add(TimeDistributed(MaxPooling2D((2, 2))))\n",
    "\n",
    "#2nd Conv2D group\n",
    "model9.add(TimeDistributed(Conv2D(32, (3, 3) , padding='same', activation='relu'), input_shape=Input_shape))\n",
    "model9.add(TimeDistributed(BatchNormalization()))\n",
    "model9.add(TimeDistributed(Conv2D(32, (3, 3) , padding='same', activation='relu')))\n",
    "model9.add(TimeDistributed(BatchNormalization()))\n",
    "model9.add(TimeDistributed(MaxPooling2D((2, 2))))\n",
    "\n",
    "#3rd Conv2D group\n",
    "model9.add(TimeDistributed(Conv2D(64, (3, 3) , padding='same', activation='relu'), input_shape=Input_shape))\n",
    "model9.add(TimeDistributed(BatchNormalization()))\n",
    "model9.add(TimeDistributed(Conv2D(64, (3, 3) , padding='same', activation='relu')))\n",
    "model9.add(TimeDistributed(BatchNormalization()))\n",
    "model9.add(TimeDistributed(MaxPooling2D((2, 2))))\n",
    "\n",
    "#FC layer group\n",
    "model9.add(TimeDistributed(Flatten()))\n",
    "#LSTM\n",
    "model9.add(LSTM(128, return_sequences=False, dropout=0.5))\n",
    "model9.add(Dense(64,activation='relu'))\n",
    "model9.add(Dropout(0.5))\n",
    "\n",
    "#Softmax Layer\n",
    "model9.add(Dense(5, activation='softmax'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "time_distributed_17 (TimeDis (None, 15, 80, 80, 16)    448       \n",
      "_________________________________________________________________\n",
      "time_distributed_18 (TimeDis (None, 15, 80, 80, 16)    64        \n",
      "_________________________________________________________________\n",
      "time_distributed_19 (TimeDis (None, 15, 80, 80, 16)    2320      \n",
      "_________________________________________________________________\n",
      "time_distributed_20 (TimeDis (None, 15, 80, 80, 16)    64        \n",
      "_________________________________________________________________\n",
      "time_distributed_21 (TimeDis (None, 15, 40, 40, 16)    0         \n",
      "_________________________________________________________________\n",
      "time_distributed_22 (TimeDis (None, 15, 40, 40, 32)    4640      \n",
      "_________________________________________________________________\n",
      "time_distributed_23 (TimeDis (None, 15, 40, 40, 32)    128       \n",
      "_________________________________________________________________\n",
      "time_distributed_24 (TimeDis (None, 15, 40, 40, 32)    9248      \n",
      "_________________________________________________________________\n",
      "time_distributed_25 (TimeDis (None, 15, 40, 40, 32)    128       \n",
      "_________________________________________________________________\n",
      "time_distributed_26 (TimeDis (None, 15, 20, 20, 32)    0         \n",
      "_________________________________________________________________\n",
      "time_distributed_27 (TimeDis (None, 15, 20, 20, 64)    18496     \n",
      "_________________________________________________________________\n",
      "time_distributed_28 (TimeDis (None, 15, 20, 20, 64)    256       \n",
      "_________________________________________________________________\n",
      "time_distributed_29 (TimeDis (None, 15, 20, 20, 64)    36928     \n",
      "_________________________________________________________________\n",
      "time_distributed_30 (TimeDis (None, 15, 20, 20, 64)    256       \n",
      "_________________________________________________________________\n",
      "time_distributed_31 (TimeDis (None, 15, 10, 10, 64)    0         \n",
      "_________________________________________________________________\n",
      "time_distributed_32 (TimeDis (None, 15, 6400)          0         \n",
      "_________________________________________________________________\n",
      "lstm_2 (LSTM)                (None, 128)               3342848   \n",
      "_________________________________________________________________\n",
      "dense_20 (Dense)             (None, 64)                8256      \n",
      "_________________________________________________________________\n",
      "dropout_12 (Dropout)         (None, 64)                0         \n",
      "_________________________________________________________________\n",
      "dense_21 (Dense)             (None, 5)                 325       \n",
      "=================================================================\n",
      "Total params: 3,424,405\n",
      "Trainable params: 3,423,957\n",
      "Non-trainable params: 448\n",
      "_________________________________________________________________\n",
      "None\n"
     ]
    }
   ],
   "source": [
    "optimiser = Adam() #write your optimizer\n",
    "model9.compile(optimizer=optimiser, loss='categorical_crossentropy', metrics=['categorical_accuracy'])\n",
    "print (model9.summary())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Conv2D + RNN (GRU) Models"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    " ## Model 10"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [],
   "source": [
    "Input_shape = (15,80,80,3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Model10\n",
    "model10 = Sequential()\n",
    "\n",
    "#1st Conv2D group\n",
    "model10.add(TimeDistributed(Conv2D(32, (3, 3) , padding='same', activation='relu'), input_shape=Input_shape))\n",
    "model10.add(TimeDistributed(BatchNormalization()))\n",
    "model10.add(TimeDistributed(Conv2D(32, (3, 3) , padding='same', activation='relu')))\n",
    "model10.add(TimeDistributed(BatchNormalization()))\n",
    "model10.add(TimeDistributed(MaxPooling2D((2, 2))))\n",
    "\n",
    "#2nd Conv2D group\n",
    "model10.add(TimeDistributed(Conv2D(64, (3, 3) , padding='same', activation='relu'), input_shape=Input_shape))\n",
    "model10.add(TimeDistributed(BatchNormalization()))\n",
    "model10.add(TimeDistributed(Conv2D(64, (3, 3) , padding='same', activation='relu')))\n",
    "model10.add(TimeDistributed(BatchNormalization()))\n",
    "model10.add(TimeDistributed(MaxPooling2D((2, 2))))\n",
    "\n",
    "#3rd Conv2D group\n",
    "model10.add(TimeDistributed(Conv2D(128, (3, 3) , padding='same', activation='relu'), input_shape=Input_shape))\n",
    "model10.add(TimeDistributed(BatchNormalization()))\n",
    "model10.add(TimeDistributed(Conv2D(128, (3, 3) , padding='same', activation='relu')))\n",
    "model10.add(TimeDistributed(BatchNormalization()))\n",
    "model10.add(TimeDistributed(MaxPooling2D((2, 2))))\n",
    "\n",
    "#FC layer group\n",
    "model10.add(TimeDistributed(Flatten()))\n",
    "#GRU\n",
    "model10.add(GRU(128, return_sequences=False, dropout=0.5))\n",
    "model10.add(Dense(64,activation='relu'))\n",
    "model10.add(Dropout(0.5))\n",
    "\n",
    "#Softmax layer\n",
    "model10.add(Dense(5, activation='softmax'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "time_distributed_33 (TimeDis (None, 15, 80, 80, 32)    896       \n",
      "_________________________________________________________________\n",
      "time_distributed_34 (TimeDis (None, 15, 80, 80, 32)    128       \n",
      "_________________________________________________________________\n",
      "time_distributed_35 (TimeDis (None, 15, 80, 80, 32)    9248      \n",
      "_________________________________________________________________\n",
      "time_distributed_36 (TimeDis (None, 15, 80, 80, 32)    128       \n",
      "_________________________________________________________________\n",
      "time_distributed_37 (TimeDis (None, 15, 40, 40, 32)    0         \n",
      "_________________________________________________________________\n",
      "time_distributed_38 (TimeDis (None, 15, 40, 40, 64)    18496     \n",
      "_________________________________________________________________\n",
      "time_distributed_39 (TimeDis (None, 15, 40, 40, 64)    256       \n",
      "_________________________________________________________________\n",
      "time_distributed_40 (TimeDis (None, 15, 40, 40, 64)    36928     \n",
      "_________________________________________________________________\n",
      "time_distributed_41 (TimeDis (None, 15, 40, 40, 64)    256       \n",
      "_________________________________________________________________\n",
      "time_distributed_42 (TimeDis (None, 15, 20, 20, 64)    0         \n",
      "_________________________________________________________________\n",
      "time_distributed_43 (TimeDis (None, 15, 20, 20, 128)   73856     \n",
      "_________________________________________________________________\n",
      "time_distributed_44 (TimeDis (None, 15, 20, 20, 128)   512       \n",
      "_________________________________________________________________\n",
      "time_distributed_45 (TimeDis (None, 15, 20, 20, 128)   147584    \n",
      "_________________________________________________________________\n",
      "time_distributed_46 (TimeDis (None, 15, 20, 20, 128)   512       \n",
      "_________________________________________________________________\n",
      "time_distributed_47 (TimeDis (None, 15, 10, 10, 128)   0         \n",
      "_________________________________________________________________\n",
      "time_distributed_48 (TimeDis (None, 15, 12800)         0         \n",
      "_________________________________________________________________\n",
      "gru_1 (GRU)                  (None, 128)               4964736   \n",
      "_________________________________________________________________\n",
      "dense_22 (Dense)             (None, 64)                8256      \n",
      "_________________________________________________________________\n",
      "dropout_13 (Dropout)         (None, 64)                0         \n",
      "_________________________________________________________________\n",
      "dense_23 (Dense)             (None, 5)                 325       \n",
      "=================================================================\n",
      "Total params: 5,262,117\n",
      "Trainable params: 5,261,221\n",
      "Non-trainable params: 896\n",
      "_________________________________________________________________\n",
      "None\n"
     ]
    }
   ],
   "source": [
    "optimiser = Adam() #write your optimizer\n",
    "model10.compile(optimizer=optimiser, loss='categorical_crossentropy', metrics=['categorical_accuracy'])\n",
    "print (model10.summary())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    " ## Model 11"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Model11\n",
    "model11 = Sequential()\n",
    "\n",
    "#1st Conv2D group\n",
    "model11.add(TimeDistributed(Conv2D(16, (3, 3) , padding='same', activation='relu'), input_shape=Input_shape))\n",
    "model11.add(TimeDistributed(BatchNormalization()))\n",
    "model11.add(TimeDistributed(Conv2D(16, (3, 3) , padding='same', activation='relu')))\n",
    "model11.add(TimeDistributed(BatchNormalization()))\n",
    "model11.add(TimeDistributed(MaxPooling2D((2, 2))))\n",
    "\n",
    "#2nd Conv2D group\n",
    "model11.add(TimeDistributed(Conv2D(32, (3, 3) , padding='same', activation='relu'), input_shape=Input_shape))\n",
    "model11.add(TimeDistributed(BatchNormalization()))\n",
    "model11.add(TimeDistributed(Conv2D(32, (3, 3) , padding='same', activation='relu')))\n",
    "model11.add(TimeDistributed(BatchNormalization()))\n",
    "model11.add(TimeDistributed(MaxPooling2D((2, 2))))\n",
    "\n",
    "#3rd Conv2D group\n",
    "model11.add(TimeDistributed(Conv2D(64, (3, 3) , padding='same', activation='relu'), input_shape=Input_shape))\n",
    "model11.add(TimeDistributed(BatchNormalization()))\n",
    "model11.add(TimeDistributed(Conv2D(64, (3, 3) , padding='same', activation='relu')))\n",
    "model11.add(TimeDistributed(BatchNormalization()))\n",
    "model11.add(TimeDistributed(MaxPooling2D((2, 2))))\n",
    "\n",
    "#FC layer group\n",
    "model11.add(TimeDistributed(Flatten()))\n",
    "#GRU\n",
    "model11.add(GRU(128, return_sequences=False, dropout=0.5))\n",
    "model11.add(Dense(64,activation='relu'))\n",
    "model11.add(Dropout(0.5))\n",
    "\n",
    "#Softmax Layer\n",
    "model11.add(Dense(5, activation='softmax'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "time_distributed_49 (TimeDis (None, 15, 80, 80, 16)    448       \n",
      "_________________________________________________________________\n",
      "time_distributed_50 (TimeDis (None, 15, 80, 80, 16)    64        \n",
      "_________________________________________________________________\n",
      "time_distributed_51 (TimeDis (None, 15, 80, 80, 16)    2320      \n",
      "_________________________________________________________________\n",
      "time_distributed_52 (TimeDis (None, 15, 80, 80, 16)    64        \n",
      "_________________________________________________________________\n",
      "time_distributed_53 (TimeDis (None, 15, 40, 40, 16)    0         \n",
      "_________________________________________________________________\n",
      "time_distributed_54 (TimeDis (None, 15, 40, 40, 32)    4640      \n",
      "_________________________________________________________________\n",
      "time_distributed_55 (TimeDis (None, 15, 40, 40, 32)    128       \n",
      "_________________________________________________________________\n",
      "time_distributed_56 (TimeDis (None, 15, 40, 40, 32)    9248      \n",
      "_________________________________________________________________\n",
      "time_distributed_57 (TimeDis (None, 15, 40, 40, 32)    128       \n",
      "_________________________________________________________________\n",
      "time_distributed_58 (TimeDis (None, 15, 20, 20, 32)    0         \n",
      "_________________________________________________________________\n",
      "time_distributed_59 (TimeDis (None, 15, 20, 20, 64)    18496     \n",
      "_________________________________________________________________\n",
      "time_distributed_60 (TimeDis (None, 15, 20, 20, 64)    256       \n",
      "_________________________________________________________________\n",
      "time_distributed_61 (TimeDis (None, 15, 20, 20, 64)    36928     \n",
      "_________________________________________________________________\n",
      "time_distributed_62 (TimeDis (None, 15, 20, 20, 64)    256       \n",
      "_________________________________________________________________\n",
      "time_distributed_63 (TimeDis (None, 15, 10, 10, 64)    0         \n",
      "_________________________________________________________________\n",
      "time_distributed_64 (TimeDis (None, 15, 6400)          0         \n",
      "_________________________________________________________________\n",
      "gru_2 (GRU)                  (None, 128)               2507136   \n",
      "_________________________________________________________________\n",
      "dense_24 (Dense)             (None, 64)                8256      \n",
      "_________________________________________________________________\n",
      "dropout_14 (Dropout)         (None, 64)                0         \n",
      "_________________________________________________________________\n",
      "dense_25 (Dense)             (None, 5)                 325       \n",
      "=================================================================\n",
      "Total params: 2,588,693\n",
      "Trainable params: 2,588,245\n",
      "Non-trainable params: 448\n",
      "_________________________________________________________________\n",
      "None\n"
     ]
    }
   ],
   "source": [
    "optimiser = Adam() #write your optimizer\n",
    "model11.compile(optimizer=optimiser, loss='categorical_crossentropy', metrics=['categorical_accuracy'])\n",
    "print (model11.summary())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Transfer Learning Models"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Non-Trainable Layers"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [],
   "source": [
    "input_shape= (15,80,80,3)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Model 12"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Downloading data from https://github.com/fchollet/deep-learning-models/releases/download/v0.6/mobilenet_1_0_224_tf_no_top.h5\n",
      "17227776/17225924 [==============================] - 2s 0us/step\n"
     ]
    }
   ],
   "source": [
    "# Model12: Transfer Learning with LSTM :-\n",
    "\n",
    "from keras.applications import mobilenet\n",
    "mobilenet_transfer = mobilenet.MobileNet(weights='imagenet', include_top=False)\n",
    "\n",
    "model12 = Sequential()\n",
    "model12.add(TimeDistributed(mobilenet_transfer, input_shape=input_shape))\n",
    "        \n",
    "for layer in model12.layers:\n",
    "    layer.trainable = False\n",
    "        \n",
    "model12.add(TimeDistributed(BatchNormalization()))\n",
    "model12.add(TimeDistributed(MaxPooling2D((2, 2))))\n",
    "\n",
    "model12.add(TimeDistributed(Flatten()))\n",
    "model12.add(LSTM(128,return_sequences=False, dropout=0.5))\n",
    "model12.add(Dense(64,activation='relu'))\n",
    "model12.add(Dropout(0.5))\n",
    "\n",
    "#Softmax Layer\n",
    "model12.add(Dense(5, activation='softmax'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "time_distributed_65 (TimeDis (None, 15, 2, 2, 1024)    3228864   \n",
      "_________________________________________________________________\n",
      "time_distributed_66 (TimeDis (None, 15, 2, 2, 1024)    4096      \n",
      "_________________________________________________________________\n",
      "time_distributed_67 (TimeDis (None, 15, 1, 1, 1024)    0         \n",
      "_________________________________________________________________\n",
      "time_distributed_68 (TimeDis (None, 15, 1024)          0         \n",
      "_________________________________________________________________\n",
      "lstm_3 (LSTM)                (None, 128)               590336    \n",
      "_________________________________________________________________\n",
      "dense_26 (Dense)             (None, 64)                8256      \n",
      "_________________________________________________________________\n",
      "dropout_15 (Dropout)         (None, 64)                0         \n",
      "_________________________________________________________________\n",
      "dense_27 (Dense)             (None, 5)                 325       \n",
      "=================================================================\n",
      "Total params: 3,831,877\n",
      "Trainable params: 600,965\n",
      "Non-trainable params: 3,230,912\n",
      "_________________________________________________________________\n",
      "None\n"
     ]
    }
   ],
   "source": [
    "optimiser = Adam() #write your optimizer\n",
    "model12.compile(optimizer=optimiser, loss='categorical_crossentropy', metrics=['categorical_accuracy'])\n",
    "print (model12.summary())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Model 13"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Model13: Transfer Learning with GRU :-\n",
    "from keras.applications import mobilenet\n",
    "mobilenet_transfer = mobilenet.MobileNet(weights='imagenet', include_top=False)\n",
    "\n",
    "model13 = Sequential()\n",
    "model13.add(TimeDistributed(mobilenet_transfer, input_shape=input_shape))\n",
    "        \n",
    "for layer in model13.layers:\n",
    "    layer.trainable = False\n",
    "        \n",
    "model13.add(TimeDistributed(BatchNormalization()))\n",
    "model13.add(TimeDistributed(MaxPooling2D((2, 2))))\n",
    "\n",
    "model13.add(TimeDistributed(Flatten()))\n",
    "\n",
    "#model13.add(Dropout(0.5))\n",
    "model13.add(GRU(128,return_sequences=False, dropout=0.5))\n",
    "model13.add(Dense(64,activation='relu'))\n",
    "model13.add(Dropout(0.5))\n",
    "model13.add(Dense(5, activation='softmax'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "time_distributed_69 (TimeDis (None, 15, 2, 2, 1024)    3228864   \n",
      "_________________________________________________________________\n",
      "time_distributed_70 (TimeDis (None, 15, 2, 2, 1024)    4096      \n",
      "_________________________________________________________________\n",
      "time_distributed_71 (TimeDis (None, 15, 1, 1, 1024)    0         \n",
      "_________________________________________________________________\n",
      "time_distributed_72 (TimeDis (None, 15, 1024)          0         \n",
      "_________________________________________________________________\n",
      "gru_3 (GRU)                  (None, 128)               442752    \n",
      "_________________________________________________________________\n",
      "dense_28 (Dense)             (None, 64)                8256      \n",
      "_________________________________________________________________\n",
      "dropout_16 (Dropout)         (None, 64)                0         \n",
      "_________________________________________________________________\n",
      "dense_29 (Dense)             (None, 5)                 325       \n",
      "=================================================================\n",
      "Total params: 3,684,293\n",
      "Trainable params: 453,381\n",
      "Non-trainable params: 3,230,912\n",
      "_________________________________________________________________\n",
      "None\n"
     ]
    }
   ],
   "source": [
    "optimiser = Adam() #write your optimizer\n",
    "model13.compile(optimizer=optimiser, loss='categorical_crossentropy', metrics=['categorical_accuracy'])\n",
    "print (model13.summary())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Trainable Layers"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [],
   "source": [
    "input_shape= (15,80,80,3)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Model 14"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Model14: Transfer Learning with LSTM :-\n",
    "\n",
    "from keras.applications import mobilenet\n",
    "mobilenet_transfer = mobilenet.MobileNet(weights='imagenet', include_top=False)\n",
    "\n",
    "model14 = Sequential()\n",
    "model14.add(TimeDistributed(mobilenet_transfer, input_shape=input_shape))\n",
    "             \n",
    "model14.add(TimeDistributed(BatchNormalization()))\n",
    "model14.add(TimeDistributed(MaxPooling2D((2, 2))))\n",
    "\n",
    "model14.add(TimeDistributed(Flatten()))\n",
    "model14.add(LSTM(128,return_sequences=False, dropout=0.5))\n",
    "model14.add(Dense(64,activation='relu'))\n",
    "model14.add(Dropout(0.5))\n",
    "\n",
    "#Softmax Layer\n",
    "model14.add(Dense(5, activation='softmax'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "time_distributed_73 (TimeDis (None, 15, 2, 2, 1024)    3228864   \n",
      "_________________________________________________________________\n",
      "time_distributed_74 (TimeDis (None, 15, 2, 2, 1024)    4096      \n",
      "_________________________________________________________________\n",
      "time_distributed_75 (TimeDis (None, 15, 1, 1, 1024)    0         \n",
      "_________________________________________________________________\n",
      "time_distributed_76 (TimeDis (None, 15, 1024)          0         \n",
      "_________________________________________________________________\n",
      "lstm_4 (LSTM)                (None, 128)               590336    \n",
      "_________________________________________________________________\n",
      "dense_30 (Dense)             (None, 64)                8256      \n",
      "_________________________________________________________________\n",
      "dropout_17 (Dropout)         (None, 64)                0         \n",
      "_________________________________________________________________\n",
      "dense_31 (Dense)             (None, 5)                 325       \n",
      "=================================================================\n",
      "Total params: 3,831,877\n",
      "Trainable params: 3,807,941\n",
      "Non-trainable params: 23,936\n",
      "_________________________________________________________________\n",
      "None\n"
     ]
    }
   ],
   "source": [
    "optimiser = Adam() #write your optimizer\n",
    "model14.compile(optimizer=optimiser, loss='categorical_crossentropy', metrics=['categorical_accuracy'])\n",
    "print (model14.summary())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Model 15"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Model15: Transfer Learning with GRU :-\n",
    "\n",
    "from keras.applications import mobilenet\n",
    "mobilenet_transfer = mobilenet.MobileNet(weights='imagenet', include_top=False)\n",
    "\n",
    "model15 = Sequential()\n",
    "model15.add(TimeDistributed(mobilenet_transfer, input_shape=input_shape))\n",
    "        \n",
    "model15.add(TimeDistributed(BatchNormalization()))\n",
    "model15.add(TimeDistributed(MaxPooling2D((2, 2))))\n",
    "\n",
    "model15.add(TimeDistributed(Flatten()))\n",
    "\n",
    "#model15.add(Dropout(0.5))\n",
    "model15.add(GRU(128,return_sequences=False, dropout=0.5))\n",
    "model15.add(Dense(64,activation='relu'))\n",
    "model15.add(Dropout(0.5))\n",
    "model15.add(Dense(5, activation='softmax'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "time_distributed_77 (TimeDis (None, 15, 2, 2, 1024)    3228864   \n",
      "_________________________________________________________________\n",
      "time_distributed_78 (TimeDis (None, 15, 2, 2, 1024)    4096      \n",
      "_________________________________________________________________\n",
      "time_distributed_79 (TimeDis (None, 15, 1, 1, 1024)    0         \n",
      "_________________________________________________________________\n",
      "time_distributed_80 (TimeDis (None, 15, 1024)          0         \n",
      "_________________________________________________________________\n",
      "gru_4 (GRU)                  (None, 128)               442752    \n",
      "_________________________________________________________________\n",
      "dense_32 (Dense)             (None, 64)                8256      \n",
      "_________________________________________________________________\n",
      "dropout_18 (Dropout)         (None, 64)                0         \n",
      "_________________________________________________________________\n",
      "dense_33 (Dense)             (None, 5)                 325       \n",
      "=================================================================\n",
      "Total params: 3,684,293\n",
      "Trainable params: 3,660,357\n",
      "Non-trainable params: 23,936\n",
      "_________________________________________________________________\n",
      "None\n"
     ]
    }
   ],
   "source": [
    "optimiser = Adam() #write your optimizer\n",
    "model15.compile(optimizer=optimiser, loss='categorical_crossentropy', metrics=['categorical_accuracy'])\n",
    "print (model15.summary())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_generator = generator(train_path, train_doc, batch_size)\n",
    "val_generator = generator(val_path, val_doc, batch_size)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [],
   "source": [
    "model_name = 'model_init' + '_' + str(curr_dt_time).replace(' ','').replace(':','_') + '/'\n",
    "    \n",
    "if not os.path.exists(model_name):\n",
    "    os.mkdir(model_name)\n",
    "        \n",
    "filepath = model_name + 'model-{epoch:05d}-{loss:.5f}-{categorical_accuracy:.5f}-{val_loss:.5f}-{val_categorical_accuracy:.5f}.h5'\n",
    "\n",
    "checkpoint = ModelCheckpoint(filepath, monitor='val_loss', verbose=1, save_best_only=False, save_weights_only=False, mode='auto', period=1)\n",
    "\n",
    "LR = ReduceLROnPlateau(monitor='val_loss', factor=0.5, patience=2, cooldown=1, verbose=1)# write the REducelronplateau code here\n",
    "callbacks_list = [checkpoint, LR]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {},
   "outputs": [],
   "source": [
    "if (num_train_sequences%batch_size) == 0:\n",
    "    steps_per_epoch = int(num_train_sequences/batch_size)\n",
    "else:\n",
    "    steps_per_epoch = (num_train_sequences//batch_size) + 1\n",
    "\n",
    "if (num_val_sequences%batch_size) == 0:\n",
    "    validation_steps = int(num_val_sequences/batch_size)\n",
    "else:\n",
    "    validation_steps = (num_val_sequences//batch_size) + 1"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Let us now fit the model. This will start training the model and with the help of the checkpoints, you'll be able to save the model at the end of each epoch."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Source path =  ./Project_data/val ; batch size = Source path =  ./Project_data/train ; batch size = 32\n",
      "32\n",
      "Epoch 1/30\n",
      "21/21 [==============================] - 92s 4s/step - loss: 3.8068 - categorical_accuracy: 0.2909 - val_loss: 1.9448 - val_categorical_accuracy: 0.2500\n",
      "\n",
      "Epoch 00001: saving model to model_init_2020-12-2711_18_53.639796/model-00001-3.84132-0.28959-1.94481-0.25000.h5\n",
      "Epoch 2/30\n",
      "21/21 [==============================] - 18s 847ms/step - loss: 1.3026 - categorical_accuracy: 0.4410 - val_loss: 1.5686 - val_categorical_accuracy: 0.1875\n",
      "\n",
      "Epoch 00002: saving model to model_init_2020-12-2711_18_53.639796/model-00002-1.30258-0.44099-1.56859-0.18750.h5\n",
      "Epoch 3/30\n",
      "21/21 [==============================] - 15s 730ms/step - loss: 0.9688 - categorical_accuracy: 0.6093 - val_loss: 1.2303 - val_categorical_accuracy: 0.5000\n",
      "\n",
      "Epoch 00003: saving model to model_init_2020-12-2711_18_53.639796/model-00003-0.97714-0.60656-1.23029-0.50000.h5\n",
      "Epoch 4/30\n",
      "21/21 [==============================] - 14s 655ms/step - loss: 1.0384 - categorical_accuracy: 0.5764 - val_loss: 0.9896 - val_categorical_accuracy: 0.5000\n",
      "\n",
      "Epoch 00004: saving model to model_init_2020-12-2711_18_53.639796/model-00004-1.03842-0.57644-0.98960-0.50000.h5\n",
      "Epoch 5/30\n",
      "21/21 [==============================] - 12s 585ms/step - loss: 0.8281 - categorical_accuracy: 0.6807 - val_loss: 0.7636 - val_categorical_accuracy: 0.6875\n",
      "\n",
      "Epoch 00005: saving model to model_init_2020-12-2711_18_53.639796/model-00005-0.82807-0.68067-0.76361-0.68750.h5\n",
      "Epoch 6/30\n",
      "21/21 [==============================] - 12s 589ms/step - loss: 0.7717 - categorical_accuracy: 0.6947 - val_loss: 0.9459 - val_categorical_accuracy: 0.6250\n",
      "\n",
      "Epoch 00006: saving model to model_init_2020-12-2711_18_53.639796/model-00006-0.77167-0.69468-0.94586-0.62500.h5\n",
      "Epoch 7/30\n",
      "21/21 [==============================] - 13s 630ms/step - loss: 0.6278 - categorical_accuracy: 0.7535 - val_loss: 0.5625 - val_categorical_accuracy: 0.6875\n",
      "\n",
      "Epoch 00007: saving model to model_init_2020-12-2711_18_53.639796/model-00007-0.62783-0.75350-0.56246-0.68750.h5\n",
      "Epoch 8/30\n",
      "21/21 [==============================] - 12s 588ms/step - loss: 0.6524 - categorical_accuracy: 0.7451 - val_loss: 0.6836 - val_categorical_accuracy: 0.7500\n",
      "\n",
      "Epoch 00008: saving model to model_init_2020-12-2711_18_53.639796/model-00008-0.65243-0.74510-0.68362-0.75000.h5\n",
      "Epoch 9/30\n",
      "21/21 [==============================] - 13s 620ms/step - loss: 0.5488 - categorical_accuracy: 0.8011 - val_loss: 1.2116 - val_categorical_accuracy: 0.6250\n",
      "\n",
      "Epoch 00009: saving model to model_init_2020-12-2711_18_53.639796/model-00009-0.54883-0.80112-1.21165-0.62500.h5\n",
      "\n",
      "Epoch 00009: ReduceLROnPlateau reducing learning rate to 0.0005000000237487257.\n",
      "Epoch 10/30\n",
      "21/21 [==============================] - 13s 612ms/step - loss: 0.4971 - categorical_accuracy: 0.8207 - val_loss: 0.9272 - val_categorical_accuracy: 0.7500\n",
      "\n",
      "Epoch 00010: saving model to model_init_2020-12-2711_18_53.639796/model-00010-0.49713-0.82073-0.92723-0.75000.h5\n",
      "Epoch 11/30\n",
      "21/21 [==============================] - 13s 606ms/step - loss: 0.3819 - categorical_accuracy: 0.8543 - val_loss: 0.7707 - val_categorical_accuracy: 0.6250\n",
      "\n",
      "Epoch 00011: saving model to model_init_2020-12-2711_18_53.639796/model-00011-0.38186-0.85434-0.77074-0.62500.h5\n",
      "\n",
      "Epoch 00011: ReduceLROnPlateau reducing learning rate to 0.0002500000118743628.\n",
      "Epoch 12/30\n",
      "21/21 [==============================] - 13s 606ms/step - loss: 0.2976 - categorical_accuracy: 0.9020 - val_loss: 0.3937 - val_categorical_accuracy: 0.8125\n",
      "\n",
      "Epoch 00012: saving model to model_init_2020-12-2711_18_53.639796/model-00012-0.29762-0.90196-0.39367-0.81250.h5\n",
      "Epoch 13/30\n",
      "21/21 [==============================] - 13s 613ms/step - loss: 0.3137 - categorical_accuracy: 0.8880 - val_loss: 0.3339 - val_categorical_accuracy: 0.9375\n",
      "\n",
      "Epoch 00013: saving model to model_init_2020-12-2711_18_53.639796/model-00013-0.31373-0.88796-0.33391-0.93750.h5\n",
      "Epoch 14/30\n",
      "21/21 [==============================] - 12s 590ms/step - loss: 0.2806 - categorical_accuracy: 0.9160 - val_loss: 1.3192 - val_categorical_accuracy: 0.6875\n",
      "\n",
      "Epoch 00014: saving model to model_init_2020-12-2711_18_53.639796/model-00014-0.28064-0.91597-1.31922-0.68750.h5\n",
      "Epoch 15/30\n",
      "21/21 [==============================] - 13s 624ms/step - loss: 0.2646 - categorical_accuracy: 0.9076 - val_loss: 0.5033 - val_categorical_accuracy: 0.8750\n",
      "\n",
      "Epoch 00015: saving model to model_init_2020-12-2711_18_53.639796/model-00015-0.26457-0.90756-0.50330-0.87500.h5\n",
      "\n",
      "Epoch 00015: ReduceLROnPlateau reducing learning rate to 0.0001250000059371814.\n",
      "Epoch 16/30\n",
      "21/21 [==============================] - 13s 631ms/step - loss: 0.2381 - categorical_accuracy: 0.9272 - val_loss: 0.8088 - val_categorical_accuracy: 0.6875\n",
      "\n",
      "Epoch 00016: saving model to model_init_2020-12-2711_18_53.639796/model-00016-0.23813-0.92717-0.80876-0.68750.h5\n",
      "Epoch 17/30\n",
      "21/21 [==============================] - 16s 763ms/step - loss: 0.2626 - categorical_accuracy: 0.9188 - val_loss: 0.4547 - val_categorical_accuracy: 0.9375\n",
      "\n",
      "Epoch 00017: saving model to model_init_2020-12-2711_18_53.639796/model-00017-0.26258-0.91877-0.45465-0.93750.h5\n",
      "\n",
      "Epoch 00017: ReduceLROnPlateau reducing learning rate to 6.25000029685907e-05.\n",
      "Epoch 18/30\n",
      "21/21 [==============================] - 15s 697ms/step - loss: 0.2387 - categorical_accuracy: 0.9048 - val_loss: 0.6117 - val_categorical_accuracy: 0.7500\n",
      "\n",
      "Epoch 00018: saving model to model_init_2020-12-2711_18_53.639796/model-00018-0.23867-0.90476-0.61171-0.75000.h5\n",
      "Epoch 19/30\n",
      "21/21 [==============================] - 17s 787ms/step - loss: 0.2123 - categorical_accuracy: 0.9272 - val_loss: 0.3245 - val_categorical_accuracy: 0.7500\n",
      "\n",
      "Epoch 00019: saving model to model_init_2020-12-2711_18_53.639796/model-00019-0.21227-0.92717-0.32449-0.75000.h5\n",
      "Epoch 20/30\n",
      "21/21 [==============================] - 12s 571ms/step - loss: 0.1998 - categorical_accuracy: 0.9384 - val_loss: 0.7484 - val_categorical_accuracy: 0.8125\n",
      "\n",
      "Epoch 00020: saving model to model_init_2020-12-2711_18_53.639796/model-00020-0.19983-0.93838-0.74844-0.81250.h5\n",
      "Epoch 21/30\n",
      "21/21 [==============================] - 12s 564ms/step - loss: 0.1389 - categorical_accuracy: 0.9748 - val_loss: 0.6973 - val_categorical_accuracy: 0.7500\n",
      "\n",
      "Epoch 00021: saving model to model_init_2020-12-2711_18_53.639796/model-00021-0.13888-0.97479-0.69727-0.75000.h5\n",
      "\n",
      "Epoch 00021: ReduceLROnPlateau reducing learning rate to 3.125000148429535e-05.\n",
      "Epoch 22/30\n",
      "21/21 [==============================] - 11s 520ms/step - loss: 0.2049 - categorical_accuracy: 0.9300 - val_loss: 0.5961 - val_categorical_accuracy: 0.8125\n",
      "\n",
      "Epoch 00022: saving model to model_init_2020-12-2711_18_53.639796/model-00022-0.20488-0.92997-0.59612-0.81250.h5\n",
      "Epoch 23/30\n",
      "21/21 [==============================] - 11s 545ms/step - loss: 0.1647 - categorical_accuracy: 0.9580 - val_loss: 0.4769 - val_categorical_accuracy: 0.8750\n",
      "\n",
      "Epoch 00023: saving model to model_init_2020-12-2711_18_53.639796/model-00023-0.16474-0.95798-0.47686-0.87500.h5\n",
      "\n",
      "Epoch 00023: ReduceLROnPlateau reducing learning rate to 1.5625000742147677e-05.\n",
      "Epoch 24/30\n",
      "21/21 [==============================] - 11s 534ms/step - loss: 0.1736 - categorical_accuracy: 0.9524 - val_loss: 0.4934 - val_categorical_accuracy: 0.8125\n",
      "\n",
      "Epoch 00024: saving model to model_init_2020-12-2711_18_53.639796/model-00024-0.17359-0.95238-0.49341-0.81250.h5\n",
      "Epoch 25/30\n",
      "21/21 [==============================] - 11s 535ms/step - loss: 0.1814 - categorical_accuracy: 0.9384 - val_loss: 0.4191 - val_categorical_accuracy: 0.8125\n",
      "\n",
      "Epoch 00025: saving model to model_init_2020-12-2711_18_53.639796/model-00025-0.18142-0.93838-0.41911-0.81250.h5\n",
      "\n",
      "Epoch 00025: ReduceLROnPlateau reducing learning rate to 7.812500371073838e-06.\n",
      "Epoch 26/30\n",
      "21/21 [==============================] - 12s 556ms/step - loss: 0.1685 - categorical_accuracy: 0.9608 - val_loss: 0.4875 - val_categorical_accuracy: 0.8125\n",
      "\n",
      "Epoch 00026: saving model to model_init_2020-12-2711_18_53.639796/model-00026-0.16845-0.96078-0.48754-0.81250.h5\n",
      "Epoch 27/30\n",
      "21/21 [==============================] - 11s 522ms/step - loss: 0.2204 - categorical_accuracy: 0.9188 - val_loss: 0.1793 - val_categorical_accuracy: 0.8750\n",
      "\n",
      "Epoch 00027: saving model to model_init_2020-12-2711_18_53.639796/model-00027-0.22041-0.91877-0.17926-0.87500.h5\n",
      "Epoch 28/30\n",
      "21/21 [==============================] - 11s 539ms/step - loss: 0.1817 - categorical_accuracy: 0.9440 - val_loss: 1.1144 - val_categorical_accuracy: 0.6875\n",
      "\n",
      "Epoch 00028: saving model to model_init_2020-12-2711_18_53.639796/model-00028-0.18166-0.94398-1.11445-0.68750.h5\n",
      "Epoch 29/30\n",
      "21/21 [==============================] - 11s 541ms/step - loss: 0.1858 - categorical_accuracy: 0.9384 - val_loss: 0.3817 - val_categorical_accuracy: 0.8750\n",
      "\n",
      "Epoch 00029: saving model to model_init_2020-12-2711_18_53.639796/model-00029-0.18575-0.93838-0.38173-0.87500.h5\n",
      "\n",
      "Epoch 00029: ReduceLROnPlateau reducing learning rate to 3.906250185536919e-06.\n",
      "Epoch 30/30\n",
      "21/21 [==============================] - 11s 534ms/step - loss: 0.1680 - categorical_accuracy: 0.9552 - val_loss: 0.2070 - val_categorical_accuracy: 0.8750\n",
      "\n",
      "Epoch 00030: saving model to model_init_2020-12-2711_18_53.639796/model-00030-0.16796-0.95518-0.20696-0.87500.h5\n"
     ]
    }
   ],
   "source": [
    "history1= model1.fit_generator(train_generator, steps_per_epoch=steps_per_epoch, epochs=num_epochs, verbose=1, \n",
    "                    callbacks=callbacks_list, validation_data=val_generator, \n",
    "                    validation_steps=validation_steps, class_weight=None, workers=1, initial_epoch=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/30\n",
      "21/21 [==============================] - 12s 578ms/step - loss: 7.6726 - categorical_accuracy: 0.2717 - val_loss: 11.7609 - val_categorical_accuracy: 0.2500\n",
      "\n",
      "Epoch 00001: saving model to model_init_2020-12-2711_18_53.639796/model-00001-7.67262-0.27171-11.76090-0.25000.h5\n",
      "Epoch 2/30\n",
      "21/21 [==============================] - 11s 511ms/step - loss: 4.3527 - categorical_accuracy: 0.2969 - val_loss: 1.4953 - val_categorical_accuracy: 0.1875\n",
      "\n",
      "Epoch 00002: saving model to model_init_2020-12-2711_18_53.639796/model-00002-4.35272-0.29692-1.49534-0.18750.h5\n",
      "Epoch 3/30\n",
      "21/21 [==============================] - 11s 542ms/step - loss: 1.3281 - categorical_accuracy: 0.4090 - val_loss: 1.3895 - val_categorical_accuracy: 0.5000\n",
      "\n",
      "Epoch 00003: saving model to model_init_2020-12-2711_18_53.639796/model-00003-1.32813-0.40896-1.38948-0.50000.h5\n",
      "Epoch 4/30\n",
      "21/21 [==============================] - 11s 522ms/step - loss: 1.2102 - categorical_accuracy: 0.4762 - val_loss: 0.7866 - val_categorical_accuracy: 0.6875\n",
      "\n",
      "Epoch 00004: saving model to model_init_2020-12-2711_18_53.639796/model-00004-1.21018-0.47619-0.78663-0.68750.h5\n",
      "Epoch 5/30\n",
      "21/21 [==============================] - 11s 548ms/step - loss: 1.0655 - categorical_accuracy: 0.5714 - val_loss: 1.0619 - val_categorical_accuracy: 0.6250\n",
      "\n",
      "Epoch 00005: saving model to model_init_2020-12-2711_18_53.639796/model-00005-1.06550-0.57143-1.06186-0.62500.h5\n",
      "Epoch 6/30\n",
      "21/21 [==============================] - 11s 504ms/step - loss: 0.9990 - categorical_accuracy: 0.6078 - val_loss: 0.8027 - val_categorical_accuracy: 0.7500\n",
      "\n",
      "Epoch 00006: saving model to model_init_2020-12-2711_18_53.639796/model-00006-0.99901-0.60784-0.80274-0.75000.h5\n",
      "\n",
      "Epoch 00006: ReduceLROnPlateau reducing learning rate to 0.0005000000237487257.\n",
      "Epoch 7/30\n",
      "21/21 [==============================] - 11s 521ms/step - loss: 0.8076 - categorical_accuracy: 0.6891 - val_loss: 0.9893 - val_categorical_accuracy: 0.6250\n",
      "\n",
      "Epoch 00007: saving model to model_init_2020-12-2711_18_53.639796/model-00007-0.80756-0.68908-0.98931-0.62500.h5\n",
      "Epoch 8/30\n",
      "21/21 [==============================] - 11s 545ms/step - loss: 0.7599 - categorical_accuracy: 0.6891 - val_loss: 0.9272 - val_categorical_accuracy: 0.5625\n",
      "\n",
      "Epoch 00008: saving model to model_init_2020-12-2711_18_53.639796/model-00008-0.75992-0.68908-0.92721-0.56250.h5\n",
      "\n",
      "Epoch 00008: ReduceLROnPlateau reducing learning rate to 0.0002500000118743628.\n",
      "Epoch 9/30\n",
      "21/21 [==============================] - 12s 548ms/step - loss: 0.7275 - categorical_accuracy: 0.7143 - val_loss: 0.9634 - val_categorical_accuracy: 0.6250\n",
      "\n",
      "Epoch 00009: saving model to model_init_2020-12-2711_18_53.639796/model-00009-0.72752-0.71429-0.96342-0.62500.h5\n",
      "Epoch 10/30\n",
      "21/21 [==============================] - 11s 540ms/step - loss: 0.6745 - categorical_accuracy: 0.7339 - val_loss: 0.6449 - val_categorical_accuracy: 0.7500\n",
      "\n",
      "Epoch 00010: saving model to model_init_2020-12-2711_18_53.639796/model-00010-0.67446-0.73389-0.64492-0.75000.h5\n",
      "Epoch 11/30\n",
      "21/21 [==============================] - 11s 532ms/step - loss: 0.6225 - categorical_accuracy: 0.7871 - val_loss: 1.1300 - val_categorical_accuracy: 0.6250\n",
      "\n",
      "Epoch 00011: saving model to model_init_2020-12-2711_18_53.639796/model-00011-0.62250-0.78711-1.13002-0.62500.h5\n",
      "Epoch 12/30\n",
      "21/21 [==============================] - 11s 542ms/step - loss: 0.5820 - categorical_accuracy: 0.7955 - val_loss: 0.7323 - val_categorical_accuracy: 0.7500\n",
      "\n",
      "Epoch 00012: saving model to model_init_2020-12-2711_18_53.639796/model-00012-0.58196-0.79552-0.73226-0.75000.h5\n",
      "\n",
      "Epoch 00012: ReduceLROnPlateau reducing learning rate to 0.0001250000059371814.\n",
      "Epoch 13/30\n",
      "21/21 [==============================] - 11s 534ms/step - loss: 0.5254 - categorical_accuracy: 0.7983 - val_loss: 0.8548 - val_categorical_accuracy: 0.8125\n",
      "\n",
      "Epoch 00013: saving model to model_init_2020-12-2711_18_53.639796/model-00013-0.52542-0.79832-0.85478-0.81250.h5\n",
      "Epoch 14/30\n",
      "21/21 [==============================] - 11s 545ms/step - loss: 0.5578 - categorical_accuracy: 0.7927 - val_loss: 1.1340 - val_categorical_accuracy: 0.5625\n",
      "\n",
      "Epoch 00014: saving model to model_init_2020-12-2711_18_53.639796/model-00014-0.55782-0.79272-1.13397-0.56250.h5\n",
      "\n",
      "Epoch 00014: ReduceLROnPlateau reducing learning rate to 6.25000029685907e-05.\n",
      "Epoch 15/30\n",
      "21/21 [==============================] - 10s 489ms/step - loss: 0.5224 - categorical_accuracy: 0.8011 - val_loss: 0.9746 - val_categorical_accuracy: 0.6875\n",
      "\n",
      "Epoch 00015: saving model to model_init_2020-12-2711_18_53.639796/model-00015-0.52242-0.80112-0.97462-0.68750.h5\n",
      "Epoch 16/30\n",
      "21/21 [==============================] - 11s 535ms/step - loss: 0.4606 - categorical_accuracy: 0.8487 - val_loss: 0.6764 - val_categorical_accuracy: 0.6250\n",
      "\n",
      "Epoch 00016: saving model to model_init_2020-12-2711_18_53.639796/model-00016-0.46058-0.84874-0.67644-0.62500.h5\n",
      "\n",
      "Epoch 00016: ReduceLROnPlateau reducing learning rate to 3.125000148429535e-05.\n",
      "Epoch 17/30\n",
      "21/21 [==============================] - 11s 542ms/step - loss: 0.4678 - categorical_accuracy: 0.8235 - val_loss: 0.5095 - val_categorical_accuracy: 0.8750\n",
      "\n",
      "Epoch 00017: saving model to model_init_2020-12-2711_18_53.639796/model-00017-0.46777-0.82353-0.50953-0.87500.h5\n",
      "Epoch 18/30\n",
      "21/21 [==============================] - 11s 546ms/step - loss: 0.4940 - categorical_accuracy: 0.8319 - val_loss: 0.6443 - val_categorical_accuracy: 0.7500\n",
      "\n",
      "Epoch 00018: saving model to model_init_2020-12-2711_18_53.639796/model-00018-0.49398-0.83193-0.64435-0.75000.h5\n",
      "Epoch 19/30\n",
      "21/21 [==============================] - 11s 506ms/step - loss: 0.4260 - categorical_accuracy: 0.8768 - val_loss: 0.3638 - val_categorical_accuracy: 0.8750\n",
      "\n",
      "Epoch 00019: saving model to model_init_2020-12-2711_18_53.639796/model-00019-0.42598-0.87675-0.36377-0.87500.h5\n",
      "Epoch 20/30\n",
      "21/21 [==============================] - 11s 544ms/step - loss: 0.5066 - categorical_accuracy: 0.8207 - val_loss: 0.9694 - val_categorical_accuracy: 0.6875\n",
      "\n",
      "Epoch 00020: saving model to model_init_2020-12-2711_18_53.639796/model-00020-0.50664-0.82073-0.96936-0.68750.h5\n",
      "Epoch 21/30\n",
      "21/21 [==============================] - 11s 536ms/step - loss: 0.4409 - categorical_accuracy: 0.8655 - val_loss: 0.8175 - val_categorical_accuracy: 0.7500\n",
      "\n",
      "Epoch 00021: saving model to model_init_2020-12-2711_18_53.639796/model-00021-0.44089-0.86555-0.81749-0.75000.h5\n",
      "\n",
      "Epoch 00021: ReduceLROnPlateau reducing learning rate to 1.5625000742147677e-05.\n",
      "Epoch 22/30\n",
      "21/21 [==============================] - 10s 494ms/step - loss: 0.4568 - categorical_accuracy: 0.8319 - val_loss: 0.5735 - val_categorical_accuracy: 0.7500\n",
      "\n",
      "Epoch 00022: saving model to model_init_2020-12-2711_18_53.639796/model-00022-0.45676-0.83193-0.57347-0.75000.h5\n",
      "Epoch 23/30\n",
      "21/21 [==============================] - 12s 550ms/step - loss: 0.4924 - categorical_accuracy: 0.8375 - val_loss: 0.4556 - val_categorical_accuracy: 0.8125\n",
      "\n",
      "Epoch 00023: saving model to model_init_2020-12-2711_18_53.639796/model-00023-0.49236-0.83754-0.45557-0.81250.h5\n",
      "\n",
      "Epoch 00023: ReduceLROnPlateau reducing learning rate to 7.812500371073838e-06.\n",
      "Epoch 24/30\n",
      "21/21 [==============================] - 11s 526ms/step - loss: 0.4531 - categorical_accuracy: 0.8123 - val_loss: 0.4977 - val_categorical_accuracy: 0.8125\n",
      "\n",
      "Epoch 00024: saving model to model_init_2020-12-2711_18_53.639796/model-00024-0.45307-0.81232-0.49766-0.81250.h5\n",
      "Epoch 25/30\n",
      "21/21 [==============================] - 12s 549ms/step - loss: 0.4395 - categorical_accuracy: 0.8543 - val_loss: 0.4256 - val_categorical_accuracy: 0.8125\n",
      "\n",
      "Epoch 00025: saving model to model_init_2020-12-2711_18_53.639796/model-00025-0.43953-0.85434-0.42557-0.81250.h5\n",
      "\n",
      "Epoch 00025: ReduceLROnPlateau reducing learning rate to 3.906250185536919e-06.\n",
      "Epoch 26/30\n",
      "21/21 [==============================] - 11s 525ms/step - loss: 0.4456 - categorical_accuracy: 0.8375 - val_loss: 0.6068 - val_categorical_accuracy: 0.7500\n",
      "\n",
      "Epoch 00026: saving model to model_init_2020-12-2711_18_53.639796/model-00026-0.44562-0.83754-0.60678-0.75000.h5\n",
      "Epoch 27/30\n",
      "21/21 [==============================] - 12s 554ms/step - loss: 0.4368 - categorical_accuracy: 0.8543 - val_loss: 0.8804 - val_categorical_accuracy: 0.6250\n",
      "\n",
      "Epoch 00027: saving model to model_init_2020-12-2711_18_53.639796/model-00027-0.43676-0.85434-0.88036-0.62500.h5\n",
      "\n",
      "Epoch 00027: ReduceLROnPlateau reducing learning rate to 1.9531250927684596e-06.\n",
      "Epoch 28/30\n",
      "21/21 [==============================] - 11s 547ms/step - loss: 0.4589 - categorical_accuracy: 0.8319 - val_loss: 0.5499 - val_categorical_accuracy: 0.8125\n",
      "\n",
      "Epoch 00028: saving model to model_init_2020-12-2711_18_53.639796/model-00028-0.45888-0.83193-0.54985-0.81250.h5\n",
      "Epoch 29/30\n",
      "21/21 [==============================] - 11s 510ms/step - loss: 0.4520 - categorical_accuracy: 0.8431 - val_loss: 0.4968 - val_categorical_accuracy: 0.8125\n",
      "\n",
      "Epoch 00029: saving model to model_init_2020-12-2711_18_53.639796/model-00029-0.45201-0.84314-0.49676-0.81250.h5\n",
      "\n",
      "Epoch 00029: ReduceLROnPlateau reducing learning rate to 9.765625463842298e-07.\n",
      "Epoch 30/30\n",
      "21/21 [==============================] - 11s 534ms/step - loss: 0.4218 - categorical_accuracy: 0.8375 - val_loss: 0.5421 - val_categorical_accuracy: 0.7500\n",
      "\n",
      "Epoch 00030: saving model to model_init_2020-12-2711_18_53.639796/model-00030-0.42177-0.83754-0.54209-0.75000.h5\n"
     ]
    }
   ],
   "source": [
    "history2= model2.fit_generator(train_generator, steps_per_epoch=steps_per_epoch, epochs=num_epochs, verbose=1, \n",
    "                    callbacks=callbacks_list, validation_data=val_generator, \n",
    "                    validation_steps=validation_steps, class_weight=None, workers=1, initial_epoch=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/30\n",
      "21/21 [==============================] - 12s 584ms/step - loss: 11.1055 - categorical_accuracy: 0.2269 - val_loss: 11.7389 - val_categorical_accuracy: 0.2500\n",
      "\n",
      "Epoch 00001: saving model to model_init_2020-12-2711_18_53.639796/model-00001-11.10555-0.22689-11.73892-0.25000.h5\n",
      "Epoch 2/30\n",
      "21/21 [==============================] - 11s 523ms/step - loss: 12.1274 - categorical_accuracy: 0.2129 - val_loss: 13.0960 - val_categorical_accuracy: 0.1875\n",
      "\n",
      "Epoch 00002: saving model to model_init_2020-12-2711_18_53.639796/model-00002-12.12738-0.21289-13.09595-0.18750.h5\n",
      "Epoch 3/30\n",
      "21/21 [==============================] - 11s 526ms/step - loss: 11.2773 - categorical_accuracy: 0.2577 - val_loss: 8.3435 - val_categorical_accuracy: 0.3125\n",
      "\n",
      "Epoch 00003: saving model to model_init_2020-12-2711_18_53.639796/model-00003-11.27725-0.25770-8.34349-0.31250.h5\n",
      "Epoch 4/30\n",
      "21/21 [==============================] - 11s 523ms/step - loss: 10.6700 - categorical_accuracy: 0.2857 - val_loss: 12.0683 - val_categorical_accuracy: 0.2500\n",
      "\n",
      "Epoch 00004: saving model to model_init_2020-12-2711_18_53.639796/model-00004-10.67001-0.28571-12.06834-0.25000.h5\n",
      "Epoch 5/30\n",
      "21/21 [==============================] - 12s 566ms/step - loss: 6.9564 - categorical_accuracy: 0.2521 - val_loss: 10.5134 - val_categorical_accuracy: 0.0625\n",
      "\n",
      "Epoch 00005: saving model to model_init_2020-12-2711_18_53.639796/model-00005-6.95643-0.25210-10.51336-0.06250.h5\n",
      "\n",
      "Epoch 00005: ReduceLROnPlateau reducing learning rate to 0.0005000000237487257.\n",
      "Epoch 6/30\n",
      "21/21 [==============================] - 12s 551ms/step - loss: 1.6123 - categorical_accuracy: 0.2353 - val_loss: 2.4171 - val_categorical_accuracy: 0.1875\n",
      "\n",
      "Epoch 00006: saving model to model_init_2020-12-2711_18_53.639796/model-00006-1.61235-0.23529-2.41707-0.18750.h5\n",
      "Epoch 7/30\n",
      "21/21 [==============================] - 12s 550ms/step - loss: 1.4764 - categorical_accuracy: 0.2997 - val_loss: 3.2582 - val_categorical_accuracy: 0.0625\n",
      "\n",
      "Epoch 00007: saving model to model_init_2020-12-2711_18_53.639796/model-00007-1.47641-0.29972-3.25823-0.06250.h5\n",
      "Epoch 8/30\n",
      "21/21 [==============================] - 11s 517ms/step - loss: 1.3629 - categorical_accuracy: 0.3978 - val_loss: 1.2733 - val_categorical_accuracy: 0.6250\n",
      "\n",
      "Epoch 00008: saving model to model_init_2020-12-2711_18_53.639796/model-00008-1.36294-0.39776-1.27326-0.62500.h5\n",
      "Epoch 9/30\n",
      "21/21 [==============================] - 12s 589ms/step - loss: 1.2516 - categorical_accuracy: 0.4482 - val_loss: 2.4376 - val_categorical_accuracy: 0.3750\n",
      "\n",
      "Epoch 00009: saving model to model_init_2020-12-2711_18_53.639796/model-00009-1.25155-0.44818-2.43760-0.37500.h5\n",
      "Epoch 10/30\n",
      "21/21 [==============================] - 11s 519ms/step - loss: 1.1440 - categorical_accuracy: 0.5322 - val_loss: 3.7053 - val_categorical_accuracy: 0.1875\n",
      "\n",
      "Epoch 00010: saving model to model_init_2020-12-2711_18_53.639796/model-00010-1.14399-0.53221-3.70534-0.18750.h5\n",
      "\n",
      "Epoch 00010: ReduceLROnPlateau reducing learning rate to 0.0002500000118743628.\n",
      "Epoch 11/30\n",
      "21/21 [==============================] - 12s 593ms/step - loss: 1.0664 - categorical_accuracy: 0.5602 - val_loss: 1.8779 - val_categorical_accuracy: 0.3125\n",
      "\n",
      "Epoch 00011: saving model to model_init_2020-12-2711_18_53.639796/model-00011-1.06641-0.56022-1.87794-0.31250.h5\n",
      "Epoch 12/30\n",
      "21/21 [==============================] - 11s 547ms/step - loss: 1.0027 - categorical_accuracy: 0.5854 - val_loss: 1.5537 - val_categorical_accuracy: 0.4375\n",
      "\n",
      "Epoch 00012: saving model to model_init_2020-12-2711_18_53.639796/model-00012-1.00271-0.58543-1.55371-0.43750.h5\n",
      "\n",
      "Epoch 00012: ReduceLROnPlateau reducing learning rate to 0.0001250000059371814.\n",
      "Epoch 13/30\n",
      "21/21 [==============================] - 12s 583ms/step - loss: 0.8578 - categorical_accuracy: 0.6611 - val_loss: 1.6665 - val_categorical_accuracy: 0.4375\n",
      "\n",
      "Epoch 00013: saving model to model_init_2020-12-2711_18_53.639796/model-00013-0.85779-0.66106-1.66647-0.43750.h5\n",
      "Epoch 14/30\n",
      "21/21 [==============================] - 12s 569ms/step - loss: 0.8305 - categorical_accuracy: 0.6947 - val_loss: 1.2471 - val_categorical_accuracy: 0.5000\n",
      "\n",
      "Epoch 00014: saving model to model_init_2020-12-2711_18_53.639796/model-00014-0.83049-0.69468-1.24705-0.50000.h5\n",
      "Epoch 15/30\n",
      "21/21 [==============================] - 11s 528ms/step - loss: 0.7820 - categorical_accuracy: 0.7087 - val_loss: 1.4034 - val_categorical_accuracy: 0.2500\n",
      "\n",
      "Epoch 00015: saving model to model_init_2020-12-2711_18_53.639796/model-00015-0.78198-0.70868-1.40340-0.25000.h5\n",
      "Epoch 16/30\n",
      "21/21 [==============================] - 12s 549ms/step - loss: 0.8420 - categorical_accuracy: 0.6471 - val_loss: 1.0352 - val_categorical_accuracy: 0.5625\n",
      "\n",
      "Epoch 00016: saving model to model_init_2020-12-2711_18_53.639796/model-00016-0.84205-0.64706-1.03517-0.56250.h5\n",
      "Epoch 17/30\n",
      "21/21 [==============================] - 12s 548ms/step - loss: 0.8464 - categorical_accuracy: 0.6499 - val_loss: 0.9845 - val_categorical_accuracy: 0.6250\n",
      "\n",
      "Epoch 00017: saving model to model_init_2020-12-2711_18_53.639796/model-00017-0.84641-0.64986-0.98455-0.62500.h5\n",
      "Epoch 18/30\n",
      "21/21 [==============================] - 12s 554ms/step - loss: 0.7648 - categorical_accuracy: 0.7087 - val_loss: 0.8604 - val_categorical_accuracy: 0.8125\n",
      "\n",
      "Epoch 00018: saving model to model_init_2020-12-2711_18_53.639796/model-00018-0.76483-0.70868-0.86041-0.81250.h5\n",
      "Epoch 19/30\n",
      "21/21 [==============================] - 12s 567ms/step - loss: 0.6655 - categorical_accuracy: 0.7647 - val_loss: 1.1395 - val_categorical_accuracy: 0.6250\n",
      "\n",
      "Epoch 00019: saving model to model_init_2020-12-2711_18_53.639796/model-00019-0.66547-0.76471-1.13948-0.62500.h5\n",
      "Epoch 20/30\n",
      "21/21 [==============================] - 12s 550ms/step - loss: 0.6725 - categorical_accuracy: 0.7563 - val_loss: 0.7109 - val_categorical_accuracy: 0.6875\n",
      "\n",
      "Epoch 00020: saving model to model_init_2020-12-2711_18_53.639796/model-00020-0.67250-0.75630-0.71093-0.68750.h5\n",
      "Epoch 21/30\n",
      "21/21 [==============================] - 12s 567ms/step - loss: 0.6877 - categorical_accuracy: 0.7535 - val_loss: 0.8297 - val_categorical_accuracy: 0.6875\n",
      "\n",
      "Epoch 00021: saving model to model_init_2020-12-2711_18_53.639796/model-00021-0.68771-0.75350-0.82966-0.68750.h5\n",
      "Epoch 22/30\n",
      "21/21 [==============================] - 12s 551ms/step - loss: 0.5953 - categorical_accuracy: 0.7479 - val_loss: 1.1004 - val_categorical_accuracy: 0.5000\n",
      "\n",
      "Epoch 00022: saving model to model_init_2020-12-2711_18_53.639796/model-00022-0.59535-0.74790-1.10037-0.50000.h5\n",
      "\n",
      "Epoch 00022: ReduceLROnPlateau reducing learning rate to 6.25000029685907e-05.\n",
      "Epoch 23/30\n",
      "21/21 [==============================] - 11s 529ms/step - loss: 0.5711 - categorical_accuracy: 0.7983 - val_loss: 0.7367 - val_categorical_accuracy: 0.6250\n",
      "\n",
      "Epoch 00023: saving model to model_init_2020-12-2711_18_53.639796/model-00023-0.57109-0.79832-0.73667-0.62500.h5\n",
      "Epoch 24/30\n",
      "21/21 [==============================] - 12s 584ms/step - loss: 0.6230 - categorical_accuracy: 0.7563 - val_loss: 0.2577 - val_categorical_accuracy: 0.8750\n",
      "\n",
      "Epoch 00024: saving model to model_init_2020-12-2711_18_53.639796/model-00024-0.62296-0.75630-0.25768-0.87500.h5\n",
      "Epoch 25/30\n",
      "21/21 [==============================] - 11s 542ms/step - loss: 0.5705 - categorical_accuracy: 0.7871 - val_loss: 0.9694 - val_categorical_accuracy: 0.6250\n",
      "\n",
      "Epoch 00025: saving model to model_init_2020-12-2711_18_53.639796/model-00025-0.57053-0.78711-0.96939-0.62500.h5\n",
      "Epoch 26/30\n",
      "21/21 [==============================] - 12s 561ms/step - loss: 0.5581 - categorical_accuracy: 0.7955 - val_loss: 1.0779 - val_categorical_accuracy: 0.5000\n",
      "\n",
      "Epoch 00026: saving model to model_init_2020-12-2711_18_53.639796/model-00026-0.55809-0.79552-1.07788-0.50000.h5\n",
      "\n",
      "Epoch 00026: ReduceLROnPlateau reducing learning rate to 3.125000148429535e-05.\n",
      "Epoch 27/30\n",
      "21/21 [==============================] - 11s 529ms/step - loss: 0.5375 - categorical_accuracy: 0.8123 - val_loss: 0.5735 - val_categorical_accuracy: 0.8125\n",
      "\n",
      "Epoch 00027: saving model to model_init_2020-12-2711_18_53.639796/model-00027-0.53753-0.81232-0.57352-0.81250.h5\n",
      "Epoch 28/30\n",
      "21/21 [==============================] - 12s 584ms/step - loss: 0.4771 - categorical_accuracy: 0.8543 - val_loss: 0.6780 - val_categorical_accuracy: 0.6250\n",
      "\n",
      "Epoch 00028: saving model to model_init_2020-12-2711_18_53.639796/model-00028-0.47713-0.85434-0.67804-0.62500.h5\n",
      "\n",
      "Epoch 00028: ReduceLROnPlateau reducing learning rate to 1.5625000742147677e-05.\n",
      "Epoch 29/30\n",
      "21/21 [==============================] - 11s 532ms/step - loss: 0.5373 - categorical_accuracy: 0.7899 - val_loss: 0.6486 - val_categorical_accuracy: 0.6875\n",
      "\n",
      "Epoch 00029: saving model to model_init_2020-12-2711_18_53.639796/model-00029-0.53728-0.78992-0.64864-0.68750.h5\n",
      "Epoch 30/30\n",
      "21/21 [==============================] - 11s 543ms/step - loss: 0.4881 - categorical_accuracy: 0.8291 - val_loss: 0.6588 - val_categorical_accuracy: 0.6875\n",
      "\n",
      "Epoch 00030: saving model to model_init_2020-12-2711_18_53.639796/model-00030-0.48810-0.82913-0.65876-0.68750.h5\n",
      "\n",
      "Epoch 00030: ReduceLROnPlateau reducing learning rate to 7.812500371073838e-06.\n"
     ]
    }
   ],
   "source": [
    "history3= model3.fit_generator(train_generator, steps_per_epoch=steps_per_epoch, epochs=num_epochs, verbose=1, \n",
    "                    callbacks=callbacks_list, validation_data=val_generator, \n",
    "                    validation_steps=validation_steps, class_weight=None, workers=1, initial_epoch=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/30\n",
      "21/21 [==============================] - 13s 624ms/step - loss: 9.2715 - categorical_accuracy: 0.2633 - val_loss: 11.0812 - val_categorical_accuracy: 0.3125\n",
      "\n",
      "Epoch 00001: saving model to model_init_2020-12-2711_18_53.639796/model-00001-9.27151-0.26331-11.08119-0.31250.h5\n",
      "Epoch 2/30\n",
      "21/21 [==============================] - 10s 496ms/step - loss: 9.1892 - categorical_accuracy: 0.2605 - val_loss: 5.8599 - val_categorical_accuracy: 0.3750\n",
      "\n",
      "Epoch 00002: saving model to model_init_2020-12-2711_18_53.639796/model-00002-9.18923-0.26050-5.85987-0.37500.h5\n",
      "Epoch 3/30\n",
      "21/21 [==============================] - 11s 506ms/step - loss: 5.6181 - categorical_accuracy: 0.3081 - val_loss: 8.6797 - val_categorical_accuracy: 0.3125\n",
      "\n",
      "Epoch 00003: saving model to model_init_2020-12-2711_18_53.639796/model-00003-5.61809-0.30812-8.67968-0.31250.h5\n",
      "Epoch 4/30\n",
      "21/21 [==============================] - 11s 539ms/step - loss: 1.3427 - categorical_accuracy: 0.4314 - val_loss: 5.7045 - val_categorical_accuracy: 0.4375\n",
      "\n",
      "Epoch 00004: saving model to model_init_2020-12-2711_18_53.639796/model-00004-1.34266-0.43137-5.70450-0.43750.h5\n",
      "Epoch 5/30\n",
      "21/21 [==============================] - 11s 523ms/step - loss: 1.2148 - categorical_accuracy: 0.4790 - val_loss: 4.4396 - val_categorical_accuracy: 0.3125\n",
      "\n",
      "Epoch 00005: saving model to model_init_2020-12-2711_18_53.639796/model-00005-1.21483-0.47899-4.43962-0.31250.h5\n",
      "Epoch 6/30\n",
      "21/21 [==============================] - 11s 547ms/step - loss: 1.0612 - categorical_accuracy: 0.5490 - val_loss: 2.0296 - val_categorical_accuracy: 0.4375\n",
      "\n",
      "Epoch 00006: saving model to model_init_2020-12-2711_18_53.639796/model-00006-1.06116-0.54902-2.02955-0.43750.h5\n",
      "Epoch 7/30\n",
      "21/21 [==============================] - 11s 530ms/step - loss: 0.9294 - categorical_accuracy: 0.6331 - val_loss: 0.8178 - val_categorical_accuracy: 0.7500\n",
      "\n",
      "Epoch 00007: saving model to model_init_2020-12-2711_18_53.639796/model-00007-0.92943-0.63305-0.81778-0.75000.h5\n",
      "Epoch 8/30\n",
      "21/21 [==============================] - 12s 552ms/step - loss: 0.8598 - categorical_accuracy: 0.6723 - val_loss: 0.8792 - val_categorical_accuracy: 0.6875\n",
      "\n",
      "Epoch 00008: saving model to model_init_2020-12-2711_18_53.639796/model-00008-0.85979-0.67227-0.87921-0.68750.h5\n",
      "Epoch 9/30\n",
      "21/21 [==============================] - 11s 526ms/step - loss: 0.6692 - categorical_accuracy: 0.7311 - val_loss: 1.6191 - val_categorical_accuracy: 0.5000\n",
      "\n",
      "Epoch 00009: saving model to model_init_2020-12-2711_18_53.639796/model-00009-0.66921-0.73109-1.61909-0.50000.h5\n",
      "\n",
      "Epoch 00009: ReduceLROnPlateau reducing learning rate to 0.0005000000237487257.\n",
      "Epoch 10/30\n",
      "21/21 [==============================] - 12s 550ms/step - loss: 0.7301 - categorical_accuracy: 0.7087 - val_loss: 0.7349 - val_categorical_accuracy: 0.7500\n",
      "\n",
      "Epoch 00010: saving model to model_init_2020-12-2711_18_53.639796/model-00010-0.73009-0.70868-0.73487-0.75000.h5\n",
      "Epoch 11/30\n",
      "21/21 [==============================] - 11s 523ms/step - loss: 0.5641 - categorical_accuracy: 0.7647 - val_loss: 0.7700 - val_categorical_accuracy: 0.6875\n",
      "\n",
      "Epoch 00011: saving model to model_init_2020-12-2711_18_53.639796/model-00011-0.56406-0.76471-0.76998-0.68750.h5\n",
      "Epoch 12/30\n",
      "21/21 [==============================] - 11s 540ms/step - loss: 0.5444 - categorical_accuracy: 0.8039 - val_loss: 0.9721 - val_categorical_accuracy: 0.5000\n",
      "\n",
      "Epoch 00012: saving model to model_init_2020-12-2711_18_53.639796/model-00012-0.54442-0.80392-0.97211-0.50000.h5\n",
      "\n",
      "Epoch 00012: ReduceLROnPlateau reducing learning rate to 0.0002500000118743628.\n",
      "Epoch 13/30\n",
      "21/21 [==============================] - 11s 533ms/step - loss: 0.4341 - categorical_accuracy: 0.8487 - val_loss: 0.8229 - val_categorical_accuracy: 0.8125\n",
      "\n",
      "Epoch 00013: saving model to model_init_2020-12-2711_18_53.639796/model-00013-0.43409-0.84874-0.82287-0.81250.h5\n",
      "Epoch 14/30\n",
      "21/21 [==============================] - 11s 533ms/step - loss: 0.4005 - categorical_accuracy: 0.8487 - val_loss: 0.8776 - val_categorical_accuracy: 0.6250\n",
      "\n",
      "Epoch 00014: saving model to model_init_2020-12-2711_18_53.639796/model-00014-0.40048-0.84874-0.87763-0.62500.h5\n",
      "\n",
      "Epoch 00014: ReduceLROnPlateau reducing learning rate to 0.0001250000059371814.\n",
      "Epoch 15/30\n",
      "21/21 [==============================] - 11s 537ms/step - loss: 0.3439 - categorical_accuracy: 0.8739 - val_loss: 0.8535 - val_categorical_accuracy: 0.6875\n",
      "\n",
      "Epoch 00015: saving model to model_init_2020-12-2711_18_53.639796/model-00015-0.34392-0.87395-0.85351-0.68750.h5\n",
      "Epoch 16/30\n",
      "21/21 [==============================] - 12s 569ms/step - loss: 0.4089 - categorical_accuracy: 0.8375 - val_loss: 1.4486 - val_categorical_accuracy: 0.6250\n",
      "\n",
      "Epoch 00016: saving model to model_init_2020-12-2711_18_53.639796/model-00016-0.40885-0.83754-1.44864-0.62500.h5\n",
      "\n",
      "Epoch 00016: ReduceLROnPlateau reducing learning rate to 6.25000029685907e-05.\n",
      "Epoch 17/30\n",
      "21/21 [==============================] - 11s 521ms/step - loss: 0.3391 - categorical_accuracy: 0.8768 - val_loss: 0.3396 - val_categorical_accuracy: 0.8750\n",
      "\n",
      "Epoch 00017: saving model to model_init_2020-12-2711_18_53.639796/model-00017-0.33914-0.87675-0.33957-0.87500.h5\n",
      "Epoch 18/30\n",
      "21/21 [==============================] - 11s 546ms/step - loss: 0.3478 - categorical_accuracy: 0.8880 - val_loss: 0.7714 - val_categorical_accuracy: 0.6250\n",
      "\n",
      "Epoch 00018: saving model to model_init_2020-12-2711_18_53.639796/model-00018-0.34775-0.88796-0.77137-0.62500.h5\n",
      "Epoch 19/30\n",
      "21/21 [==============================] - 11s 522ms/step - loss: 0.3473 - categorical_accuracy: 0.8824 - val_loss: 0.7954 - val_categorical_accuracy: 0.7500\n",
      "\n",
      "Epoch 00019: saving model to model_init_2020-12-2711_18_53.639796/model-00019-0.34727-0.88235-0.79539-0.75000.h5\n",
      "\n",
      "Epoch 00019: ReduceLROnPlateau reducing learning rate to 3.125000148429535e-05.\n",
      "Epoch 20/30\n",
      "21/21 [==============================] - 11s 520ms/step - loss: 0.3301 - categorical_accuracy: 0.8739 - val_loss: 0.7289 - val_categorical_accuracy: 0.8750\n",
      "\n",
      "Epoch 00020: saving model to model_init_2020-12-2711_18_53.639796/model-00020-0.33008-0.87395-0.72893-0.87500.h5\n",
      "Epoch 21/30\n",
      "21/21 [==============================] - 11s 531ms/step - loss: 0.3276 - categorical_accuracy: 0.8908 - val_loss: 0.7372 - val_categorical_accuracy: 0.8125\n",
      "\n",
      "Epoch 00021: saving model to model_init_2020-12-2711_18_53.639796/model-00021-0.32761-0.89076-0.73715-0.81250.h5\n",
      "\n",
      "Epoch 00021: ReduceLROnPlateau reducing learning rate to 1.5625000742147677e-05.\n",
      "Epoch 22/30\n",
      "21/21 [==============================] - 10s 499ms/step - loss: 0.3129 - categorical_accuracy: 0.9020 - val_loss: 0.4334 - val_categorical_accuracy: 0.8750\n",
      "\n",
      "Epoch 00022: saving model to model_init_2020-12-2711_18_53.639796/model-00022-0.31287-0.90196-0.43340-0.87500.h5\n",
      "Epoch 23/30\n",
      "21/21 [==============================] - 11s 542ms/step - loss: 0.3276 - categorical_accuracy: 0.8880 - val_loss: 0.2783 - val_categorical_accuracy: 0.9375\n",
      "\n",
      "Epoch 00023: saving model to model_init_2020-12-2711_18_53.639796/model-00023-0.32760-0.88796-0.27827-0.93750.h5\n",
      "Epoch 24/30\n",
      "21/21 [==============================] - 12s 548ms/step - loss: 0.3185 - categorical_accuracy: 0.8908 - val_loss: 1.0315 - val_categorical_accuracy: 0.7500\n",
      "\n",
      "Epoch 00024: saving model to model_init_2020-12-2711_18_53.639796/model-00024-0.31854-0.89076-1.03146-0.75000.h5\n",
      "Epoch 25/30\n",
      "21/21 [==============================] - 10s 498ms/step - loss: 0.3239 - categorical_accuracy: 0.8936 - val_loss: 0.2783 - val_categorical_accuracy: 0.9375\n",
      "\n",
      "Epoch 00025: saving model to model_init_2020-12-2711_18_53.639796/model-00025-0.32390-0.89356-0.27826-0.93750.h5\n",
      "\n",
      "Epoch 00025: ReduceLROnPlateau reducing learning rate to 7.812500371073838e-06.\n",
      "Epoch 26/30\n",
      "21/21 [==============================] - 11s 544ms/step - loss: 0.3353 - categorical_accuracy: 0.8683 - val_loss: 0.6062 - val_categorical_accuracy: 0.7500\n",
      "\n",
      "Epoch 00026: saving model to model_init_2020-12-2711_18_53.639796/model-00026-0.33529-0.86835-0.60616-0.75000.h5\n",
      "Epoch 27/30\n",
      "21/21 [==============================] - 11s 501ms/step - loss: 0.3291 - categorical_accuracy: 0.8852 - val_loss: 0.8632 - val_categorical_accuracy: 0.6875\n",
      "\n",
      "Epoch 00027: saving model to model_init_2020-12-2711_18_53.639796/model-00027-0.32914-0.88515-0.86318-0.68750.h5\n",
      "\n",
      "Epoch 00027: ReduceLROnPlateau reducing learning rate to 3.906250185536919e-06.\n",
      "Epoch 28/30\n",
      "21/21 [==============================] - 11s 532ms/step - loss: 0.2640 - categorical_accuracy: 0.9076 - val_loss: 0.6412 - val_categorical_accuracy: 0.8125\n",
      "\n",
      "Epoch 00028: saving model to model_init_2020-12-2711_18_53.639796/model-00028-0.26405-0.90756-0.64122-0.81250.h5\n",
      "Epoch 29/30\n",
      "21/21 [==============================] - 12s 556ms/step - loss: 0.3215 - categorical_accuracy: 0.8936 - val_loss: 0.5316 - val_categorical_accuracy: 0.7500\n",
      "\n",
      "Epoch 00029: saving model to model_init_2020-12-2711_18_53.639796/model-00029-0.32154-0.89356-0.53159-0.75000.h5\n",
      "\n",
      "Epoch 00029: ReduceLROnPlateau reducing learning rate to 1.9531250927684596e-06.\n",
      "Epoch 30/30\n",
      "21/21 [==============================] - 11s 534ms/step - loss: 0.3236 - categorical_accuracy: 0.8852 - val_loss: 0.5679 - val_categorical_accuracy: 0.8750\n",
      "\n",
      "Epoch 00030: saving model to model_init_2020-12-2711_18_53.639796/model-00030-0.32358-0.88515-0.56788-0.87500.h5\n"
     ]
    }
   ],
   "source": [
    "history4= model4.fit_generator(train_generator, steps_per_epoch=steps_per_epoch, epochs=num_epochs, verbose=1, \n",
    "                    callbacks=callbacks_list, validation_data=val_generator, \n",
    "                    validation_steps=validation_steps, class_weight=None, workers=1, initial_epoch=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Source path =  ./Project_data/val ; batch size = 32\n",
      "Source path =  ./Project_data/train ; batch size =Epoch 1/30\n",
      " 32\n",
      "21/21 [==============================] - 23s 1s/step - loss: 2.4084 - categorical_accuracy: 0.2937 - val_loss: 1.5069 - val_categorical_accuracy: 0.2500\n",
      "\n",
      "Epoch 00001: saving model to model_init_2020-12-2711_40_24.647928/model-00001-2.41729-0.29412-1.50690-0.25000.h5\n",
      "Epoch 2/30\n",
      "21/21 [==============================] - 15s 707ms/step - loss: 1.5415 - categorical_accuracy: 0.3685 - val_loss: 1.1205 - val_categorical_accuracy: 0.5000\n",
      "\n",
      "Epoch 00002: saving model to model_init_2020-12-2711_40_24.647928/model-00002-1.54149-0.36853-1.12053-0.50000.h5\n",
      "Epoch 3/30\n",
      "21/21 [==============================] - 13s 617ms/step - loss: 1.3041 - categorical_accuracy: 0.4223 - val_loss: 1.0976 - val_categorical_accuracy: 0.6875\n",
      "\n",
      "Epoch 00003: saving model to model_init_2020-12-2711_40_24.647928/model-00003-1.30638-0.42389-1.09756-0.68750.h5\n",
      "Epoch 4/30\n",
      "21/21 [==============================] - 13s 609ms/step - loss: 1.1736 - categorical_accuracy: 0.4737 - val_loss: 1.1004 - val_categorical_accuracy: 0.6250\n",
      "\n",
      "Epoch 00004: saving model to model_init_2020-12-2711_40_24.647928/model-00004-1.17361-0.47368-1.10037-0.62500.h5\n",
      "Epoch 5/30\n",
      "21/21 [==============================] - 12s 554ms/step - loss: 1.1133 - categorical_accuracy: 0.5350 - val_loss: 0.9290 - val_categorical_accuracy: 0.6250\n",
      "\n",
      "Epoch 00005: saving model to model_init_2020-12-2711_40_24.647928/model-00005-1.11326-0.53501-0.92897-0.62500.h5\n",
      "Epoch 6/30\n",
      "21/21 [==============================] - 11s 540ms/step - loss: 1.0319 - categorical_accuracy: 0.5742 - val_loss: 0.9649 - val_categorical_accuracy: 0.6250\n",
      "\n",
      "Epoch 00006: saving model to model_init_2020-12-2711_40_24.647928/model-00006-1.03194-0.57423-0.96488-0.62500.h5\n",
      "Epoch 7/30\n",
      "21/21 [==============================] - 12s 584ms/step - loss: 0.9176 - categorical_accuracy: 0.6611 - val_loss: 1.1965 - val_categorical_accuracy: 0.5000\n",
      "\n",
      "Epoch 00007: saving model to model_init_2020-12-2711_40_24.647928/model-00007-0.91759-0.66106-1.19653-0.50000.h5\n",
      "\n",
      "Epoch 00007: ReduceLROnPlateau reducing learning rate to 0.0005000000237487257.\n",
      "Epoch 8/30\n",
      "21/21 [==============================] - 10s 467ms/step - loss: 0.8384 - categorical_accuracy: 0.6835 - val_loss: 0.7317 - val_categorical_accuracy: 0.7500\n",
      "\n",
      "Epoch 00008: saving model to model_init_2020-12-2711_40_24.647928/model-00008-0.83837-0.68347-0.73170-0.75000.h5\n",
      "Epoch 9/30\n",
      "21/21 [==============================] - 12s 559ms/step - loss: 0.8086 - categorical_accuracy: 0.6863 - val_loss: 0.7276 - val_categorical_accuracy: 0.7500\n",
      "\n",
      "Epoch 00009: saving model to model_init_2020-12-2711_40_24.647928/model-00009-0.80857-0.68627-0.72763-0.75000.h5\n",
      "Epoch 10/30\n",
      "21/21 [==============================] - 11s 544ms/step - loss: 0.7079 - categorical_accuracy: 0.7451 - val_loss: 0.5967 - val_categorical_accuracy: 0.7500\n",
      "\n",
      "Epoch 00010: saving model to model_init_2020-12-2711_40_24.647928/model-00010-0.70793-0.74510-0.59667-0.75000.h5\n",
      "Epoch 11/30\n",
      "21/21 [==============================] - 11s 533ms/step - loss: 0.6241 - categorical_accuracy: 0.7787 - val_loss: 1.1278 - val_categorical_accuracy: 0.6250\n",
      "\n",
      "Epoch 00011: saving model to model_init_2020-12-2711_40_24.647928/model-00011-0.62408-0.77871-1.12783-0.62500.h5\n",
      "Epoch 12/30\n",
      "21/21 [==============================] - 11s 512ms/step - loss: 0.6533 - categorical_accuracy: 0.7451 - val_loss: 0.3293 - val_categorical_accuracy: 0.8750\n",
      "\n",
      "Epoch 00012: saving model to model_init_2020-12-2711_40_24.647928/model-00012-0.65330-0.74510-0.32929-0.87500.h5\n",
      "Epoch 13/30\n",
      "21/21 [==============================] - 12s 566ms/step - loss: 0.5604 - categorical_accuracy: 0.7983 - val_loss: 1.1912 - val_categorical_accuracy: 0.6875\n",
      "\n",
      "Epoch 00013: saving model to model_init_2020-12-2711_40_24.647928/model-00013-0.56042-0.79832-1.19123-0.68750.h5\n",
      "Epoch 14/30\n",
      "21/21 [==============================] - 12s 586ms/step - loss: 0.5013 - categorical_accuracy: 0.8263 - val_loss: 1.0031 - val_categorical_accuracy: 0.7500\n",
      "\n",
      "Epoch 00014: saving model to model_init_2020-12-2711_40_24.647928/model-00014-0.50135-0.82633-1.00310-0.75000.h5\n",
      "\n",
      "Epoch 00014: ReduceLROnPlateau reducing learning rate to 0.0002500000118743628.\n",
      "Epoch 15/30\n",
      "21/21 [==============================] - 11s 544ms/step - loss: 0.5071 - categorical_accuracy: 0.8291 - val_loss: 0.7273 - val_categorical_accuracy: 0.7500\n",
      "\n",
      "Epoch 00015: saving model to model_init_2020-12-2711_40_24.647928/model-00015-0.50709-0.82913-0.72731-0.75000.h5\n",
      "Epoch 16/30\n",
      "21/21 [==============================] - 11s 539ms/step - loss: 0.4453 - categorical_accuracy: 0.8431 - val_loss: 0.8408 - val_categorical_accuracy: 0.7500\n",
      "\n",
      "Epoch 00016: saving model to model_init_2020-12-2711_40_24.647928/model-00016-0.44527-0.84314-0.84084-0.75000.h5\n",
      "\n",
      "Epoch 00016: ReduceLROnPlateau reducing learning rate to 0.0001250000059371814.\n",
      "Epoch 17/30\n",
      "21/21 [==============================] - 12s 561ms/step - loss: 0.4471 - categorical_accuracy: 0.8375 - val_loss: 0.3354 - val_categorical_accuracy: 0.9375\n",
      "\n",
      "Epoch 00017: saving model to model_init_2020-12-2711_40_24.647928/model-00017-0.44708-0.83754-0.33537-0.93750.h5\n",
      "Epoch 18/30\n",
      "21/21 [==============================] - 11s 540ms/step - loss: 0.3715 - categorical_accuracy: 0.8880 - val_loss: 0.7930 - val_categorical_accuracy: 0.6875\n",
      "\n",
      "Epoch 00018: saving model to model_init_2020-12-2711_40_24.647928/model-00018-0.37153-0.88796-0.79303-0.68750.h5\n",
      "\n",
      "Epoch 00018: ReduceLROnPlateau reducing learning rate to 6.25000029685907e-05.\n",
      "Epoch 19/30\n",
      "21/21 [==============================] - 12s 553ms/step - loss: 0.4123 - categorical_accuracy: 0.8375 - val_loss: 0.5457 - val_categorical_accuracy: 0.8125\n",
      "\n",
      "Epoch 00019: saving model to model_init_2020-12-2711_40_24.647928/model-00019-0.41234-0.83754-0.54567-0.81250.h5\n",
      "Epoch 20/30\n",
      "21/21 [==============================] - 12s 559ms/step - loss: 0.3917 - categorical_accuracy: 0.8543 - val_loss: 0.5338 - val_categorical_accuracy: 0.8125\n",
      "\n",
      "Epoch 00020: saving model to model_init_2020-12-2711_40_24.647928/model-00020-0.39171-0.85434-0.53382-0.81250.h5\n",
      "\n",
      "Epoch 00020: ReduceLROnPlateau reducing learning rate to 3.125000148429535e-05.\n",
      "Epoch 21/30\n",
      "21/21 [==============================] - 11s 547ms/step - loss: 0.3605 - categorical_accuracy: 0.8683 - val_loss: 0.2935 - val_categorical_accuracy: 0.8750\n",
      "\n",
      "Epoch 00021: saving model to model_init_2020-12-2711_40_24.647928/model-00021-0.36051-0.86835-0.29346-0.87500.h5\n",
      "Epoch 22/30\n",
      "21/21 [==============================] - 11s 533ms/step - loss: 0.3957 - categorical_accuracy: 0.8515 - val_loss: 0.5428 - val_categorical_accuracy: 0.8125\n",
      "\n",
      "Epoch 00022: saving model to model_init_2020-12-2711_40_24.647928/model-00022-0.39566-0.85154-0.54279-0.81250.h5\n",
      "Epoch 23/30\n",
      "21/21 [==============================] - 12s 566ms/step - loss: 0.3112 - categorical_accuracy: 0.8992 - val_loss: 0.8650 - val_categorical_accuracy: 0.6250\n",
      "\n",
      "Epoch 00023: saving model to model_init_2020-12-2711_40_24.647928/model-00023-0.31121-0.89916-0.86496-0.62500.h5\n",
      "\n",
      "Epoch 00023: ReduceLROnPlateau reducing learning rate to 1.5625000742147677e-05.\n",
      "Epoch 24/30\n",
      "21/21 [==============================] - 11s 542ms/step - loss: 0.3369 - categorical_accuracy: 0.8655 - val_loss: 0.7662 - val_categorical_accuracy: 0.8125\n",
      "\n",
      "Epoch 00024: saving model to model_init_2020-12-2711_40_24.647928/model-00024-0.33690-0.86555-0.76621-0.81250.h5\n",
      "Epoch 25/30\n",
      "21/21 [==============================] - 12s 553ms/step - loss: 0.3266 - categorical_accuracy: 0.8964 - val_loss: 0.6435 - val_categorical_accuracy: 0.8125\n",
      "\n",
      "Epoch 00025: saving model to model_init_2020-12-2711_40_24.647928/model-00025-0.32658-0.89636-0.64355-0.81250.h5\n",
      "\n",
      "Epoch 00025: ReduceLROnPlateau reducing learning rate to 7.812500371073838e-06.\n",
      "Epoch 26/30\n",
      "21/21 [==============================] - 12s 560ms/step - loss: 0.3232 - categorical_accuracy: 0.8852 - val_loss: 0.5546 - val_categorical_accuracy: 0.8125\n",
      "\n",
      "Epoch 00026: saving model to model_init_2020-12-2711_40_24.647928/model-00026-0.32325-0.88515-0.55457-0.81250.h5\n",
      "Epoch 27/30\n",
      "21/21 [==============================] - 11s 532ms/step - loss: 0.3729 - categorical_accuracy: 0.8543 - val_loss: 0.3959 - val_categorical_accuracy: 0.8750\n",
      "\n",
      "Epoch 00027: saving model to model_init_2020-12-2711_40_24.647928/model-00027-0.37288-0.85434-0.39586-0.87500.h5\n",
      "\n",
      "Epoch 00027: ReduceLROnPlateau reducing learning rate to 3.906250185536919e-06.\n",
      "Epoch 28/30\n",
      "21/21 [==============================] - 11s 532ms/step - loss: 0.3644 - categorical_accuracy: 0.8543 - val_loss: 0.7804 - val_categorical_accuracy: 0.7500\n",
      "\n",
      "Epoch 00028: saving model to model_init_2020-12-2711_40_24.647928/model-00028-0.36437-0.85434-0.78038-0.75000.h5\n",
      "Epoch 29/30\n",
      "21/21 [==============================] - 11s 530ms/step - loss: 0.3242 - categorical_accuracy: 0.8852 - val_loss: 0.6492 - val_categorical_accuracy: 0.8125\n",
      "\n",
      "Epoch 00029: saving model to model_init_2020-12-2711_40_24.647928/model-00029-0.32423-0.88515-0.64915-0.81250.h5\n",
      "\n",
      "Epoch 00029: ReduceLROnPlateau reducing learning rate to 1.9531250927684596e-06.\n",
      "Epoch 30/30\n",
      "21/21 [==============================] - 11s 530ms/step - loss: 0.3287 - categorical_accuracy: 0.8880 - val_loss: 0.2154 - val_categorical_accuracy: 0.9375\n",
      "\n",
      "Epoch 00030: saving model to model_init_2020-12-2711_40_24.647928/model-00030-0.32867-0.88796-0.21542-0.93750.h5\n"
     ]
    }
   ],
   "source": [
    "history5= model5.fit_generator(train_generator, steps_per_epoch=steps_per_epoch, epochs=num_epochs, verbose=1, \n",
    "                    callbacks=callbacks_list, validation_data=val_generator, \n",
    "                    validation_steps=validation_steps, class_weight=None, workers=1, initial_epoch=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/30\n",
      "21/21 [==============================] - 12s 571ms/step - loss: 2.6207 - categorical_accuracy: 0.2129 - val_loss: 3.8426 - val_categorical_accuracy: 0.1875\n",
      "\n",
      "Epoch 00001: saving model to model_init_2020-12-2711_40_24.647928/model-00001-2.62074-0.21289-3.84255-0.18750.h5\n",
      "Epoch 2/30\n",
      "21/21 [==============================] - 10s 484ms/step - loss: 1.5513 - categorical_accuracy: 0.2997 - val_loss: 3.1461 - val_categorical_accuracy: 0.3750\n",
      "\n",
      "Epoch 00002: saving model to model_init_2020-12-2711_40_24.647928/model-00002-1.55129-0.29972-3.14606-0.37500.h5\n",
      "Epoch 3/30\n",
      "21/21 [==============================] - 12s 562ms/step - loss: 1.4136 - categorical_accuracy: 0.3754 - val_loss: 2.5756 - val_categorical_accuracy: 0.1250\n",
      "\n",
      "Epoch 00003: saving model to model_init_2020-12-2711_40_24.647928/model-00003-1.41356-0.37535-2.57563-0.12500.h5\n",
      "Epoch 4/30\n",
      "21/21 [==============================] - 11s 531ms/step - loss: 1.4012 - categorical_accuracy: 0.3754 - val_loss: 3.3840 - val_categorical_accuracy: 0.1875\n",
      "\n",
      "Epoch 00004: saving model to model_init_2020-12-2711_40_24.647928/model-00004-1.40116-0.37535-3.38399-0.18750.h5\n",
      "Epoch 5/30\n",
      "21/21 [==============================] - 12s 571ms/step - loss: 1.2788 - categorical_accuracy: 0.4426 - val_loss: 2.8682 - val_categorical_accuracy: 0.1875\n",
      "\n",
      "Epoch 00005: saving model to model_init_2020-12-2711_40_24.647928/model-00005-1.27884-0.44258-2.86817-0.18750.h5\n",
      "\n",
      "Epoch 00005: ReduceLROnPlateau reducing learning rate to 0.0005000000237487257.\n",
      "Epoch 6/30\n",
      "21/21 [==============================] - 11s 511ms/step - loss: 1.2917 - categorical_accuracy: 0.4398 - val_loss: 1.6411 - val_categorical_accuracy: 0.3750\n",
      "\n",
      "Epoch 00006: saving model to model_init_2020-12-2711_40_24.647928/model-00006-1.29172-0.43978-1.64115-0.37500.h5\n",
      "Epoch 7/30\n",
      "21/21 [==============================] - 12s 552ms/step - loss: 1.2119 - categorical_accuracy: 0.4370 - val_loss: 1.3923 - val_categorical_accuracy: 0.4375\n",
      "\n",
      "Epoch 00007: saving model to model_init_2020-12-2711_40_24.647928/model-00007-1.21186-0.43697-1.39234-0.43750.h5\n",
      "Epoch 8/30\n",
      "21/21 [==============================] - 12s 562ms/step - loss: 1.2268 - categorical_accuracy: 0.4622 - val_loss: 1.2167 - val_categorical_accuracy: 0.3750\n",
      "\n",
      "Epoch 00008: saving model to model_init_2020-12-2711_40_24.647928/model-00008-1.22682-0.46218-1.21669-0.37500.h5\n",
      "Epoch 9/30\n",
      "21/21 [==============================] - 12s 559ms/step - loss: 1.1379 - categorical_accuracy: 0.5070 - val_loss: 1.1791 - val_categorical_accuracy: 0.6250\n",
      "\n",
      "Epoch 00009: saving model to model_init_2020-12-2711_40_24.647928/model-00009-1.13786-0.50700-1.17914-0.62500.h5\n",
      "Epoch 10/30\n",
      "21/21 [==============================] - 12s 555ms/step - loss: 1.1269 - categorical_accuracy: 0.4986 - val_loss: 1.7225 - val_categorical_accuracy: 0.3750\n",
      "\n",
      "Epoch 00010: saving model to model_init_2020-12-2711_40_24.647928/model-00010-1.12693-0.49860-1.72253-0.37500.h5\n",
      "Epoch 11/30\n",
      "21/21 [==============================] - 12s 552ms/step - loss: 1.1069 - categorical_accuracy: 0.5490 - val_loss: 1.5242 - val_categorical_accuracy: 0.5000\n",
      "\n",
      "Epoch 00011: saving model to model_init_2020-12-2711_40_24.647928/model-00011-1.10685-0.54902-1.52416-0.50000.h5\n",
      "\n",
      "Epoch 00011: ReduceLROnPlateau reducing learning rate to 0.0002500000118743628.\n",
      "Epoch 12/30\n",
      "21/21 [==============================] - 12s 555ms/step - loss: 1.0385 - categorical_accuracy: 0.5630 - val_loss: 1.3679 - val_categorical_accuracy: 0.3750\n",
      "\n",
      "Epoch 00012: saving model to model_init_2020-12-2711_40_24.647928/model-00012-1.03847-0.56303-1.36788-0.37500.h5\n",
      "Epoch 13/30\n",
      "21/21 [==============================] - 12s 550ms/step - loss: 1.0078 - categorical_accuracy: 0.6050 - val_loss: 1.1911 - val_categorical_accuracy: 0.5625\n",
      "\n",
      "Epoch 00013: saving model to model_init_2020-12-2711_40_24.647928/model-00013-1.00784-0.60504-1.19113-0.56250.h5\n",
      "\n",
      "Epoch 00013: ReduceLROnPlateau reducing learning rate to 0.0001250000059371814.\n",
      "Epoch 14/30\n",
      "21/21 [==============================] - 12s 571ms/step - loss: 0.9666 - categorical_accuracy: 0.6275 - val_loss: 1.4513 - val_categorical_accuracy: 0.5000\n",
      "\n",
      "Epoch 00014: saving model to model_init_2020-12-2711_40_24.647928/model-00014-0.96661-0.62745-1.45134-0.50000.h5\n",
      "Epoch 15/30\n",
      "21/21 [==============================] - 11s 515ms/step - loss: 0.9910 - categorical_accuracy: 0.5826 - val_loss: 1.4684 - val_categorical_accuracy: 0.3750\n",
      "\n",
      "Epoch 00015: saving model to model_init_2020-12-2711_40_24.647928/model-00015-0.99102-0.58263-1.46836-0.37500.h5\n",
      "\n",
      "Epoch 00015: ReduceLROnPlateau reducing learning rate to 6.25000029685907e-05.\n",
      "Epoch 16/30\n",
      "21/21 [==============================] - 12s 550ms/step - loss: 0.9508 - categorical_accuracy: 0.6162 - val_loss: 1.0400 - val_categorical_accuracy: 0.6250\n",
      "\n",
      "Epoch 00016: saving model to model_init_2020-12-2711_40_24.647928/model-00016-0.95081-0.61625-1.04003-0.62500.h5\n",
      "Epoch 17/30\n",
      "21/21 [==============================] - 12s 555ms/step - loss: 0.9178 - categorical_accuracy: 0.6246 - val_loss: 0.9052 - val_categorical_accuracy: 0.5625\n",
      "\n",
      "Epoch 00017: saving model to model_init_2020-12-2711_40_24.647928/model-00017-0.91777-0.62465-0.90518-0.56250.h5\n",
      "Epoch 18/30\n",
      "21/21 [==============================] - 12s 569ms/step - loss: 0.9820 - categorical_accuracy: 0.5770 - val_loss: 1.8626 - val_categorical_accuracy: 0.3125\n",
      "\n",
      "Epoch 00018: saving model to model_init_2020-12-2711_40_24.647928/model-00018-0.98197-0.57703-1.86257-0.31250.h5\n",
      "Epoch 19/30\n",
      "21/21 [==============================] - 11s 534ms/step - loss: 0.9012 - categorical_accuracy: 0.6275 - val_loss: 1.0413 - val_categorical_accuracy: 0.6250\n",
      "\n",
      "Epoch 00019: saving model to model_init_2020-12-2711_40_24.647928/model-00019-0.90122-0.62745-1.04132-0.62500.h5\n",
      "\n",
      "Epoch 00019: ReduceLROnPlateau reducing learning rate to 3.125000148429535e-05.\n",
      "Epoch 20/30\n",
      "21/21 [==============================] - 12s 561ms/step - loss: 0.8852 - categorical_accuracy: 0.6387 - val_loss: 1.2936 - val_categorical_accuracy: 0.5625\n",
      "\n",
      "Epoch 00020: saving model to model_init_2020-12-2711_40_24.647928/model-00020-0.88517-0.63866-1.29358-0.56250.h5\n",
      "Epoch 21/30\n",
      "21/21 [==============================] - 12s 562ms/step - loss: 0.9368 - categorical_accuracy: 0.6275 - val_loss: 1.2025 - val_categorical_accuracy: 0.5625\n",
      "\n",
      "Epoch 00021: saving model to model_init_2020-12-2711_40_24.647928/model-00021-0.93681-0.62745-1.20247-0.56250.h5\n",
      "\n",
      "Epoch 00021: ReduceLROnPlateau reducing learning rate to 1.5625000742147677e-05.\n",
      "Epoch 22/30\n",
      "21/21 [==============================] - 11s 519ms/step - loss: 0.9220 - categorical_accuracy: 0.6246 - val_loss: 0.8404 - val_categorical_accuracy: 0.6875\n",
      "\n",
      "Epoch 00022: saving model to model_init_2020-12-2711_40_24.647928/model-00022-0.92201-0.62465-0.84043-0.68750.h5\n",
      "Epoch 23/30\n",
      "21/21 [==============================] - 12s 567ms/step - loss: 0.8688 - categorical_accuracy: 0.6527 - val_loss: 0.9304 - val_categorical_accuracy: 0.5625\n",
      "\n",
      "Epoch 00023: saving model to model_init_2020-12-2711_40_24.647928/model-00023-0.86881-0.65266-0.93044-0.56250.h5\n",
      "Epoch 24/30\n",
      "21/21 [==============================] - 11s 544ms/step - loss: 0.8852 - categorical_accuracy: 0.6611 - val_loss: 1.0901 - val_categorical_accuracy: 0.5000\n",
      "\n",
      "Epoch 00024: saving model to model_init_2020-12-2711_40_24.647928/model-00024-0.88522-0.66106-1.09006-0.50000.h5\n",
      "\n",
      "Epoch 00024: ReduceLROnPlateau reducing learning rate to 7.812500371073838e-06.\n",
      "Epoch 25/30\n",
      "21/21 [==============================] - 12s 576ms/step - loss: 0.8834 - categorical_accuracy: 0.6639 - val_loss: 0.6650 - val_categorical_accuracy: 0.7500\n",
      "\n",
      "Epoch 00025: saving model to model_init_2020-12-2711_40_24.647928/model-00025-0.88338-0.66387-0.66503-0.75000.h5\n",
      "Epoch 26/30\n",
      "21/21 [==============================] - 12s 551ms/step - loss: 0.8831 - categorical_accuracy: 0.6611 - val_loss: 0.9023 - val_categorical_accuracy: 0.6250\n",
      "\n",
      "Epoch 00026: saving model to model_init_2020-12-2711_40_24.647928/model-00026-0.88315-0.66106-0.90235-0.62500.h5\n",
      "Epoch 27/30\n",
      "21/21 [==============================] - 12s 569ms/step - loss: 0.8356 - categorical_accuracy: 0.6779 - val_loss: 0.9523 - val_categorical_accuracy: 0.5625\n",
      "\n",
      "Epoch 00027: saving model to model_init_2020-12-2711_40_24.647928/model-00027-0.83565-0.67787-0.95230-0.56250.h5\n",
      "\n",
      "Epoch 00027: ReduceLROnPlateau reducing learning rate to 3.906250185536919e-06.\n",
      "Epoch 28/30\n",
      "21/21 [==============================] - 12s 567ms/step - loss: 0.9341 - categorical_accuracy: 0.6022 - val_loss: 1.0554 - val_categorical_accuracy: 0.5000\n",
      "\n",
      "Epoch 00028: saving model to model_init_2020-12-2711_40_24.647928/model-00028-0.93414-0.60224-1.05535-0.50000.h5\n",
      "Epoch 29/30\n",
      "21/21 [==============================] - 11s 545ms/step - loss: 0.8733 - categorical_accuracy: 0.6583 - val_loss: 0.7565 - val_categorical_accuracy: 0.8125\n",
      "\n",
      "Epoch 00029: saving model to model_init_2020-12-2711_40_24.647928/model-00029-0.87335-0.65826-0.75650-0.81250.h5\n",
      "\n",
      "Epoch 00029: ReduceLROnPlateau reducing learning rate to 1.9531250927684596e-06.\n",
      "Epoch 30/30\n",
      "21/21 [==============================] - 12s 560ms/step - loss: 0.8606 - categorical_accuracy: 0.6779 - val_loss: 0.8170 - val_categorical_accuracy: 0.7500\n",
      "\n",
      "Epoch 00030: saving model to model_init_2020-12-2711_40_24.647928/model-00030-0.86063-0.67787-0.81703-0.75000.h5\n"
     ]
    }
   ],
   "source": [
    "history6= model6.fit_generator(train_generator, steps_per_epoch=steps_per_epoch, epochs=num_epochs, verbose=1, \n",
    "                    callbacks=callbacks_list, validation_data=val_generator, \n",
    "                    validation_steps=validation_steps, class_weight=None, workers=1, initial_epoch=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/30\n",
      "21/21 [==============================] - 14s 649ms/step - loss: 7.1450 - categorical_accuracy: 0.2381 - val_loss: 3.1092 - val_categorical_accuracy: 0.3750\n",
      "\n",
      "Epoch 00001: saving model to model_init_2020-12-2711_40_24.647928/model-00001-7.14503-0.23810-3.10924-0.37500.h5\n",
      "Epoch 2/30\n",
      "21/21 [==============================] - 9s 448ms/step - loss: 2.1482 - categorical_accuracy: 0.2493 - val_loss: 1.7200 - val_categorical_accuracy: 0.3125\n",
      "\n",
      "Epoch 00002: saving model to model_init_2020-12-2711_40_24.647928/model-00002-2.14819-0.24930-1.72002-0.31250.h5\n",
      "Epoch 3/30\n",
      "21/21 [==============================] - 12s 558ms/step - loss: 1.3425 - categorical_accuracy: 0.4118 - val_loss: 1.5124 - val_categorical_accuracy: 0.3125\n",
      "\n",
      "Epoch 00003: saving model to model_init_2020-12-2711_40_24.647928/model-00003-1.34245-0.41176-1.51238-0.31250.h5\n",
      "Epoch 4/30\n",
      "21/21 [==============================] - 12s 551ms/step - loss: 1.2750 - categorical_accuracy: 0.4202 - val_loss: 2.1827 - val_categorical_accuracy: 0.1250\n",
      "\n",
      "Epoch 00004: saving model to model_init_2020-12-2711_40_24.647928/model-00004-1.27496-0.42017-2.18273-0.12500.h5\n",
      "Epoch 5/30\n",
      "21/21 [==============================] - 11s 541ms/step - loss: 1.3036 - categorical_accuracy: 0.4538 - val_loss: 0.9481 - val_categorical_accuracy: 0.6875\n",
      "\n",
      "Epoch 00005: saving model to model_init_2020-12-2711_40_24.647928/model-00005-1.30358-0.45378-0.94810-0.68750.h5\n",
      "Epoch 6/30\n",
      "21/21 [==============================] - 12s 585ms/step - loss: 1.1954 - categorical_accuracy: 0.4902 - val_loss: 0.9702 - val_categorical_accuracy: 0.6875\n",
      "\n",
      "Epoch 00006: saving model to model_init_2020-12-2711_40_24.647928/model-00006-1.19545-0.49020-0.97018-0.68750.h5\n",
      "Epoch 7/30\n",
      "21/21 [==============================] - 11s 533ms/step - loss: 1.1544 - categorical_accuracy: 0.5322 - val_loss: 1.5901 - val_categorical_accuracy: 0.5000\n",
      "\n",
      "Epoch 00007: saving model to model_init_2020-12-2711_40_24.647928/model-00007-1.15439-0.53221-1.59006-0.50000.h5\n",
      "\n",
      "Epoch 00007: ReduceLROnPlateau reducing learning rate to 0.0005000000237487257.\n",
      "Epoch 8/30\n",
      "21/21 [==============================] - 11s 521ms/step - loss: 1.0052 - categorical_accuracy: 0.5518 - val_loss: 0.9941 - val_categorical_accuracy: 0.7500\n",
      "\n",
      "Epoch 00008: saving model to model_init_2020-12-2711_40_24.647928/model-00008-1.00518-0.55182-0.99412-0.75000.h5\n",
      "Epoch 9/30\n",
      "21/21 [==============================] - 11s 542ms/step - loss: 0.9149 - categorical_accuracy: 0.6106 - val_loss: 0.9806 - val_categorical_accuracy: 0.5625\n",
      "\n",
      "Epoch 00009: saving model to model_init_2020-12-2711_40_24.647928/model-00009-0.91491-0.61064-0.98063-0.56250.h5\n",
      "\n",
      "Epoch 00009: ReduceLROnPlateau reducing learning rate to 0.0002500000118743628.\n",
      "Epoch 10/30\n",
      "21/21 [==============================] - 11s 547ms/step - loss: 0.8698 - categorical_accuracy: 0.6723 - val_loss: 0.9707 - val_categorical_accuracy: 0.6875\n",
      "\n",
      "Epoch 00010: saving model to model_init_2020-12-2711_40_24.647928/model-00010-0.86976-0.67227-0.97066-0.68750.h5\n",
      "Epoch 11/30\n",
      "21/21 [==============================] - 11s 545ms/step - loss: 0.7734 - categorical_accuracy: 0.7059 - val_loss: 1.5471 - val_categorical_accuracy: 0.5000\n",
      "\n",
      "Epoch 00011: saving model to model_init_2020-12-2711_40_24.647928/model-00011-0.77345-0.70588-1.54710-0.50000.h5\n",
      "\n",
      "Epoch 00011: ReduceLROnPlateau reducing learning rate to 0.0001250000059371814.\n",
      "Epoch 12/30\n",
      "21/21 [==============================] - 12s 572ms/step - loss: 0.8135 - categorical_accuracy: 0.6863 - val_loss: 1.1538 - val_categorical_accuracy: 0.6875\n",
      "\n",
      "Epoch 00012: saving model to model_init_2020-12-2711_40_24.647928/model-00012-0.81349-0.68627-1.15385-0.68750.h5\n",
      "Epoch 13/30\n",
      "21/21 [==============================] - 11s 538ms/step - loss: 0.7478 - categorical_accuracy: 0.7031 - val_loss: 0.3943 - val_categorical_accuracy: 0.9375\n",
      "\n",
      "Epoch 00013: saving model to model_init_2020-12-2711_40_24.647928/model-00013-0.74778-0.70308-0.39426-0.93750.h5\n",
      "Epoch 14/30\n",
      "21/21 [==============================] - 12s 591ms/step - loss: 0.7688 - categorical_accuracy: 0.7003 - val_loss: 0.7316 - val_categorical_accuracy: 0.6250\n",
      "\n",
      "Epoch 00014: saving model to model_init_2020-12-2711_40_24.647928/model-00014-0.76885-0.70028-0.73160-0.62500.h5\n",
      "Epoch 15/30\n",
      "21/21 [==============================] - 12s 554ms/step - loss: 0.7362 - categorical_accuracy: 0.7143 - val_loss: 1.0030 - val_categorical_accuracy: 0.6250\n",
      "\n",
      "Epoch 00015: saving model to model_init_2020-12-2711_40_24.647928/model-00015-0.73620-0.71429-1.00296-0.62500.h5\n",
      "\n",
      "Epoch 00015: ReduceLROnPlateau reducing learning rate to 6.25000029685907e-05.\n",
      "Epoch 16/30\n",
      "21/21 [==============================] - 12s 580ms/step - loss: 0.7058 - categorical_accuracy: 0.7143 - val_loss: 1.0198 - val_categorical_accuracy: 0.5625\n",
      "\n",
      "Epoch 00016: saving model to model_init_2020-12-2711_40_24.647928/model-00016-0.70582-0.71429-1.01976-0.56250.h5\n",
      "Epoch 17/30\n",
      "21/21 [==============================] - 12s 588ms/step - loss: 0.7080 - categorical_accuracy: 0.7311 - val_loss: 0.9531 - val_categorical_accuracy: 0.6250\n",
      "\n",
      "Epoch 00017: saving model to model_init_2020-12-2711_40_24.647928/model-00017-0.70801-0.73109-0.95308-0.62500.h5\n",
      "\n",
      "Epoch 00017: ReduceLROnPlateau reducing learning rate to 3.125000148429535e-05.\n",
      "Epoch 18/30\n",
      "21/21 [==============================] - 11s 533ms/step - loss: 0.6551 - categorical_accuracy: 0.7507 - val_loss: 0.6238 - val_categorical_accuracy: 0.7500\n",
      "\n",
      "Epoch 00018: saving model to model_init_2020-12-2711_40_24.647928/model-00018-0.65505-0.75070-0.62376-0.75000.h5\n",
      "Epoch 19/30\n",
      "21/21 [==============================] - 11s 545ms/step - loss: 0.6638 - categorical_accuracy: 0.7339 - val_loss: 0.7391 - val_categorical_accuracy: 0.6250\n",
      "\n",
      "Epoch 00019: saving model to model_init_2020-12-2711_40_24.647928/model-00019-0.66385-0.73389-0.73908-0.62500.h5\n",
      "\n",
      "Epoch 00019: ReduceLROnPlateau reducing learning rate to 1.5625000742147677e-05.\n",
      "Epoch 20/30\n",
      "21/21 [==============================] - 12s 562ms/step - loss: 0.6630 - categorical_accuracy: 0.7479 - val_loss: 1.0236 - val_categorical_accuracy: 0.5000\n",
      "\n",
      "Epoch 00020: saving model to model_init_2020-12-2711_40_24.647928/model-00020-0.66297-0.74790-1.02360-0.50000.h5\n",
      "Epoch 21/30\n",
      "21/21 [==============================] - 12s 592ms/step - loss: 0.6529 - categorical_accuracy: 0.7507 - val_loss: 0.2362 - val_categorical_accuracy: 1.0000\n",
      "\n",
      "Epoch 00021: saving model to model_init_2020-12-2711_40_24.647928/model-00021-0.65293-0.75070-0.23620-1.00000.h5\n",
      "Epoch 22/30\n",
      "21/21 [==============================] - 11s 543ms/step - loss: 0.6913 - categorical_accuracy: 0.7367 - val_loss: 0.6227 - val_categorical_accuracy: 0.8125\n",
      "\n",
      "Epoch 00022: saving model to model_init_2020-12-2711_40_24.647928/model-00022-0.69128-0.73669-0.62272-0.81250.h5\n",
      "Epoch 23/30\n",
      "21/21 [==============================] - 12s 561ms/step - loss: 0.5869 - categorical_accuracy: 0.8095 - val_loss: 0.9490 - val_categorical_accuracy: 0.6875\n",
      "\n",
      "Epoch 00023: saving model to model_init_2020-12-2711_40_24.647928/model-00023-0.58689-0.80952-0.94902-0.68750.h5\n",
      "\n",
      "Epoch 00023: ReduceLROnPlateau reducing learning rate to 7.812500371073838e-06.\n",
      "Epoch 24/30\n",
      "21/21 [==============================] - 12s 549ms/step - loss: 0.6847 - categorical_accuracy: 0.7395 - val_loss: 0.4882 - val_categorical_accuracy: 0.8125\n",
      "\n",
      "Epoch 00024: saving model to model_init_2020-12-2711_40_24.647928/model-00024-0.68472-0.73950-0.48818-0.81250.h5\n",
      "Epoch 25/30\n",
      "21/21 [==============================] - 12s 593ms/step - loss: 0.6390 - categorical_accuracy: 0.7451 - val_loss: 0.5356 - val_categorical_accuracy: 0.7500\n",
      "\n",
      "Epoch 00025: saving model to model_init_2020-12-2711_40_24.647928/model-00025-0.63903-0.74510-0.53563-0.75000.h5\n",
      "\n",
      "Epoch 00025: ReduceLROnPlateau reducing learning rate to 3.906250185536919e-06.\n",
      "Epoch 26/30\n",
      "21/21 [==============================] - 11s 546ms/step - loss: 0.6727 - categorical_accuracy: 0.7311 - val_loss: 0.5689 - val_categorical_accuracy: 0.8750\n",
      "\n",
      "Epoch 00026: saving model to model_init_2020-12-2711_40_24.647928/model-00026-0.67267-0.73109-0.56888-0.87500.h5\n",
      "Epoch 27/30\n",
      "21/21 [==============================] - 12s 579ms/step - loss: 0.6425 - categorical_accuracy: 0.7703 - val_loss: 0.4283 - val_categorical_accuracy: 0.9375\n",
      "\n",
      "Epoch 00027: saving model to model_init_2020-12-2711_40_24.647928/model-00027-0.64249-0.77031-0.42826-0.93750.h5\n",
      "\n",
      "Epoch 00027: ReduceLROnPlateau reducing learning rate to 1.9531250927684596e-06.\n",
      "Epoch 28/30\n",
      "21/21 [==============================] - 12s 571ms/step - loss: 0.6489 - categorical_accuracy: 0.7563 - val_loss: 0.8851 - val_categorical_accuracy: 0.6250\n",
      "\n",
      "Epoch 00028: saving model to model_init_2020-12-2711_40_24.647928/model-00028-0.64888-0.75630-0.88507-0.62500.h5\n",
      "Epoch 29/30\n",
      "21/21 [==============================] - 11s 547ms/step - loss: 0.6203 - categorical_accuracy: 0.7787 - val_loss: 0.7562 - val_categorical_accuracy: 0.6875\n",
      "\n",
      "Epoch 00029: saving model to model_init_2020-12-2711_40_24.647928/model-00029-0.62034-0.77871-0.75616-0.68750.h5\n",
      "\n",
      "Epoch 00029: ReduceLROnPlateau reducing learning rate to 9.765625463842298e-07.\n",
      "Epoch 30/30\n",
      "21/21 [==============================] - 12s 585ms/step - loss: 0.6108 - categorical_accuracy: 0.7759 - val_loss: 0.6427 - val_categorical_accuracy: 0.7500\n",
      "\n",
      "Epoch 00030: saving model to model_init_2020-12-2711_40_24.647928/model-00030-0.61082-0.77591-0.64273-0.75000.h5\n"
     ]
    }
   ],
   "source": [
    "history7= model7.fit_generator(train_generator, steps_per_epoch=steps_per_epoch, epochs=num_epochs, verbose=1, \n",
    "                    callbacks=callbacks_list, validation_data=val_generator, \n",
    "                    validation_steps=validation_steps, class_weight=None, workers=1, initial_epoch=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Source path =  ./Project_data/val ; batch size = 32\n",
      "Source path =  ./Project_data/train ; batch size =Epoch 1/30\n",
      " 32\n",
      "21/21 [==============================] - 25s 1s/step - loss: 1.5872 - categorical_accuracy: 0.2740 - val_loss: 1.6007 - val_categorical_accuracy: 0.2200\n",
      "\n",
      "Epoch 00001: saving model to model_init_2020-12-2713_52_38.997171/model-00001-1.58817-0.27300-1.60070-0.22000.h5\n",
      "Epoch 2/30\n",
      "21/21 [==============================] - 13s 627ms/step - loss: 1.3877 - categorical_accuracy: 0.4244 - val_loss: 1.7744 - val_categorical_accuracy: 0.1875\n",
      "\n",
      "Epoch 00002: saving model to model_init_2020-12-2713_52_38.997171/model-00002-1.38772-0.42443-1.77445-0.18750.h5\n",
      "Epoch 3/30\n",
      "21/21 [==============================] - 12s 565ms/step - loss: 1.3029 - categorical_accuracy: 0.4686 - val_loss: 1.9531 - val_categorical_accuracy: 0.1250\n",
      "\n",
      "Epoch 00003: saving model to model_init_2020-12-2713_52_38.997171/model-00003-1.30765-0.46604-1.95312-0.12500.h5\n",
      "\n",
      "Epoch 00003: ReduceLROnPlateau reducing learning rate to 0.0005000000237487257.\n",
      "Epoch 4/30\n",
      "21/21 [==============================] - 12s 576ms/step - loss: 1.2422 - categorical_accuracy: 0.4586 - val_loss: 1.3426 - val_categorical_accuracy: 0.4375\n",
      "\n",
      "Epoch 00004: saving model to model_init_2020-12-2713_52_38.997171/model-00004-1.24221-0.45865-1.34260-0.43750.h5\n",
      "Epoch 5/30\n",
      "21/21 [==============================] - 12s 565ms/step - loss: 1.2290 - categorical_accuracy: 0.4650 - val_loss: 2.0311 - val_categorical_accuracy: 0.3125\n",
      "\n",
      "Epoch 00005: saving model to model_init_2020-12-2713_52_38.997171/model-00005-1.22898-0.46499-2.03105-0.31250.h5\n",
      "Epoch 6/30\n",
      "21/21 [==============================] - 11s 532ms/step - loss: 1.1125 - categorical_accuracy: 0.5378 - val_loss: 1.9901 - val_categorical_accuracy: 0.2500\n",
      "\n",
      "Epoch 00006: saving model to model_init_2020-12-2713_52_38.997171/model-00006-1.11249-0.53782-1.99008-0.25000.h5\n",
      "\n",
      "Epoch 00006: ReduceLROnPlateau reducing learning rate to 0.0002500000118743628.\n",
      "Epoch 7/30\n",
      "21/21 [==============================] - 11s 528ms/step - loss: 1.0300 - categorical_accuracy: 0.5518 - val_loss: 1.6933 - val_categorical_accuracy: 0.3750\n",
      "\n",
      "Epoch 00007: saving model to model_init_2020-12-2713_52_38.997171/model-00007-1.02995-0.55182-1.69325-0.37500.h5\n",
      "Epoch 8/30\n",
      "21/21 [==============================] - 12s 552ms/step - loss: 1.0506 - categorical_accuracy: 0.5630 - val_loss: 1.1408 - val_categorical_accuracy: 0.4375\n",
      "\n",
      "Epoch 00008: saving model to model_init_2020-12-2713_52_38.997171/model-00008-1.05060-0.56303-1.14081-0.43750.h5\n",
      "Epoch 9/30\n",
      "21/21 [==============================] - 11s 543ms/step - loss: 0.9474 - categorical_accuracy: 0.6443 - val_loss: 1.3462 - val_categorical_accuracy: 0.4375\n",
      "\n",
      "Epoch 00009: saving model to model_init_2020-12-2713_52_38.997171/model-00009-0.94736-0.64426-1.34623-0.43750.h5\n",
      "Epoch 10/30\n",
      "21/21 [==============================] - 11s 523ms/step - loss: 0.9418 - categorical_accuracy: 0.6387 - val_loss: 0.7734 - val_categorical_accuracy: 0.7500\n",
      "\n",
      "Epoch 00010: saving model to model_init_2020-12-2713_52_38.997171/model-00010-0.94183-0.63866-0.77336-0.75000.h5\n",
      "Epoch 11/30\n",
      "21/21 [==============================] - 11s 518ms/step - loss: 0.9451 - categorical_accuracy: 0.6246 - val_loss: 0.9596 - val_categorical_accuracy: 0.7500\n",
      "\n",
      "Epoch 00011: saving model to model_init_2020-12-2713_52_38.997171/model-00011-0.94511-0.62465-0.95957-0.75000.h5\n",
      "Epoch 12/30\n",
      "21/21 [==============================] - 12s 561ms/step - loss: 0.7619 - categorical_accuracy: 0.7507 - val_loss: 0.9215 - val_categorical_accuracy: 0.5625\n",
      "\n",
      "Epoch 00012: saving model to model_init_2020-12-2713_52_38.997171/model-00012-0.76189-0.75070-0.92151-0.56250.h5\n",
      "\n",
      "Epoch 00012: ReduceLROnPlateau reducing learning rate to 0.0001250000059371814.\n",
      "Epoch 13/30\n",
      "21/21 [==============================] - 11s 513ms/step - loss: 0.8580 - categorical_accuracy: 0.6975 - val_loss: 1.0010 - val_categorical_accuracy: 0.6875\n",
      "\n",
      "Epoch 00013: saving model to model_init_2020-12-2713_52_38.997171/model-00013-0.85799-0.69748-1.00102-0.68750.h5\n",
      "Epoch 14/30\n",
      "21/21 [==============================] - 11s 509ms/step - loss: 0.7675 - categorical_accuracy: 0.7199 - val_loss: 0.7581 - val_categorical_accuracy: 0.6875\n",
      "\n",
      "Epoch 00014: saving model to model_init_2020-12-2713_52_38.997171/model-00014-0.76747-0.71989-0.75807-0.68750.h5\n",
      "Epoch 15/30\n",
      "21/21 [==============================] - 11s 524ms/step - loss: 0.7705 - categorical_accuracy: 0.7451 - val_loss: 0.8583 - val_categorical_accuracy: 0.6875\n",
      "\n",
      "Epoch 00015: saving model to model_init_2020-12-2713_52_38.997171/model-00015-0.77052-0.74510-0.85834-0.68750.h5\n",
      "Epoch 16/30\n",
      "21/21 [==============================] - 11s 520ms/step - loss: 0.7535 - categorical_accuracy: 0.7059 - val_loss: 0.5368 - val_categorical_accuracy: 0.9375\n",
      "\n",
      "Epoch 00016: saving model to model_init_2020-12-2713_52_38.997171/model-00016-0.75347-0.70588-0.53679-0.93750.h5\n",
      "Epoch 17/30\n",
      "21/21 [==============================] - 11s 524ms/step - loss: 0.7110 - categorical_accuracy: 0.7479 - val_loss: 0.9587 - val_categorical_accuracy: 0.5000\n",
      "\n",
      "Epoch 00017: saving model to model_init_2020-12-2713_52_38.997171/model-00017-0.71098-0.74790-0.95869-0.50000.h5\n",
      "Epoch 18/30\n",
      "21/21 [==============================] - 12s 549ms/step - loss: 0.6690 - categorical_accuracy: 0.7703 - val_loss: 0.8247 - val_categorical_accuracy: 0.7500\n",
      "\n",
      "Epoch 00018: saving model to model_init_2020-12-2713_52_38.997171/model-00018-0.66904-0.77031-0.82468-0.75000.h5\n",
      "\n",
      "Epoch 00018: ReduceLROnPlateau reducing learning rate to 6.25000029685907e-05.\n",
      "Epoch 19/30\n",
      "21/21 [==============================] - 11s 509ms/step - loss: 0.6376 - categorical_accuracy: 0.7983 - val_loss: 0.8789 - val_categorical_accuracy: 0.6875\n",
      "\n",
      "Epoch 00019: saving model to model_init_2020-12-2713_52_38.997171/model-00019-0.63760-0.79832-0.87892-0.68750.h5\n",
      "Epoch 20/30\n",
      "21/21 [==============================] - 11s 537ms/step - loss: 0.6184 - categorical_accuracy: 0.8011 - val_loss: 0.7548 - val_categorical_accuracy: 0.8750\n",
      "\n",
      "Epoch 00020: saving model to model_init_2020-12-2713_52_38.997171/model-00020-0.61837-0.80112-0.75475-0.87500.h5\n",
      "\n",
      "Epoch 00020: ReduceLROnPlateau reducing learning rate to 3.125000148429535e-05.\n",
      "Epoch 21/30\n",
      "21/21 [==============================] - 11s 544ms/step - loss: 0.6172 - categorical_accuracy: 0.7815 - val_loss: 0.8681 - val_categorical_accuracy: 0.6250\n",
      "\n",
      "Epoch 00021: saving model to model_init_2020-12-2713_52_38.997171/model-00021-0.61715-0.78151-0.86811-0.62500.h5\n",
      "Epoch 22/30\n",
      "21/21 [==============================] - 11s 542ms/step - loss: 0.5843 - categorical_accuracy: 0.7927 - val_loss: 0.9182 - val_categorical_accuracy: 0.5625\n",
      "\n",
      "Epoch 00022: saving model to model_init_2020-12-2713_52_38.997171/model-00022-0.58431-0.79272-0.91823-0.56250.h5\n",
      "\n",
      "Epoch 00022: ReduceLROnPlateau reducing learning rate to 1.5625000742147677e-05.\n",
      "Epoch 23/30\n",
      "21/21 [==============================] - 11s 523ms/step - loss: 0.6045 - categorical_accuracy: 0.7871 - val_loss: 0.6787 - val_categorical_accuracy: 0.8125\n",
      "\n",
      "Epoch 00023: saving model to model_init_2020-12-2713_52_38.997171/model-00023-0.60454-0.78711-0.67869-0.81250.h5\n",
      "Epoch 24/30\n",
      "21/21 [==============================] - 11s 520ms/step - loss: 0.6033 - categorical_accuracy: 0.7927 - val_loss: 0.7643 - val_categorical_accuracy: 0.7500\n",
      "\n",
      "Epoch 00024: saving model to model_init_2020-12-2713_52_38.997171/model-00024-0.60334-0.79272-0.76433-0.75000.h5\n",
      "\n",
      "Epoch 00024: ReduceLROnPlateau reducing learning rate to 7.812500371073838e-06.\n",
      "Epoch 25/30\n",
      "21/21 [==============================] - 12s 553ms/step - loss: 0.6236 - categorical_accuracy: 0.7787 - val_loss: 0.7741 - val_categorical_accuracy: 0.7500\n",
      "\n",
      "Epoch 00025: saving model to model_init_2020-12-2713_52_38.997171/model-00025-0.62362-0.77871-0.77407-0.75000.h5\n",
      "Epoch 26/30\n",
      "21/21 [==============================] - 11s 531ms/step - loss: 0.5980 - categorical_accuracy: 0.8235 - val_loss: 0.7705 - val_categorical_accuracy: 0.7500\n",
      "\n",
      "Epoch 00026: saving model to model_init_2020-12-2713_52_38.997171/model-00026-0.59797-0.82353-0.77045-0.75000.h5\n",
      "\n",
      "Epoch 00026: ReduceLROnPlateau reducing learning rate to 3.906250185536919e-06.\n",
      "Epoch 27/30\n",
      "21/21 [==============================] - 11s 527ms/step - loss: 0.5931 - categorical_accuracy: 0.8067 - val_loss: 0.6870 - val_categorical_accuracy: 0.6875\n",
      "\n",
      "Epoch 00027: saving model to model_init_2020-12-2713_52_38.997171/model-00027-0.59307-0.80672-0.68699-0.68750.h5\n",
      "Epoch 28/30\n",
      "21/21 [==============================] - 11s 538ms/step - loss: 0.5326 - categorical_accuracy: 0.8263 - val_loss: 0.9462 - val_categorical_accuracy: 0.6875\n",
      "\n",
      "Epoch 00028: saving model to model_init_2020-12-2713_52_38.997171/model-00028-0.53257-0.82633-0.94621-0.68750.h5\n",
      "\n",
      "Epoch 00028: ReduceLROnPlateau reducing learning rate to 1.9531250927684596e-06.\n",
      "Epoch 29/30\n",
      "21/21 [==============================] - 12s 551ms/step - loss: 0.6205 - categorical_accuracy: 0.7955 - val_loss: 0.8064 - val_categorical_accuracy: 0.6875\n",
      "\n",
      "Epoch 00029: saving model to model_init_2020-12-2713_52_38.997171/model-00029-0.62052-0.79552-0.80643-0.68750.h5\n",
      "Epoch 30/30\n",
      "21/21 [==============================] - 11s 547ms/step - loss: 0.5774 - categorical_accuracy: 0.8515 - val_loss: 0.5649 - val_categorical_accuracy: 0.8125\n",
      "\n",
      "Epoch 00030: saving model to model_init_2020-12-2713_52_38.997171/model-00030-0.57744-0.85154-0.56487-0.81250.h5\n",
      "\n",
      "Epoch 00030: ReduceLROnPlateau reducing learning rate to 9.765625463842298e-07.\n"
     ]
    }
   ],
   "source": [
    "history8= model8.fit_generator(train_generator, steps_per_epoch=steps_per_epoch, epochs=num_epochs, verbose=1, \n",
    "                    callbacks=callbacks_list, validation_data=val_generator, \n",
    "                    validation_steps=validation_steps, class_weight=None, workers=1, initial_epoch=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/30\n",
      "21/21 [==============================] - 14s 674ms/step - loss: 1.6501 - categorical_accuracy: 0.2409 - val_loss: 1.7179 - val_categorical_accuracy: 0.1875\n",
      "\n",
      "Epoch 00001: saving model to model_init_2020-12-2713_37_00.524791/model-00001-1.65014-0.24090-1.71790-0.18750.h5\n",
      "Epoch 2/30\n",
      "21/21 [==============================] - 9s 420ms/step - loss: 1.4175 - categorical_accuracy: 0.3866 - val_loss: 1.3279 - val_categorical_accuracy: 0.5000\n",
      "\n",
      "Epoch 00002: saving model to model_init_2020-12-2713_37_00.524791/model-00002-1.41746-0.38655-1.32787-0.50000.h5\n",
      "Epoch 3/30\n",
      "21/21 [==============================] - 11s 522ms/step - loss: 1.3230 - categorical_accuracy: 0.4426 - val_loss: 1.1261 - val_categorical_accuracy: 0.6875\n",
      "\n",
      "Epoch 00003: saving model to model_init_2020-12-2713_37_00.524791/model-00003-1.32297-0.44258-1.12610-0.68750.h5\n",
      "Epoch 4/30\n",
      "21/21 [==============================] - 12s 559ms/step - loss: 1.2258 - categorical_accuracy: 0.4846 - val_loss: 1.1147 - val_categorical_accuracy: 0.6250\n",
      "\n",
      "Epoch 00004: saving model to model_init_2020-12-2713_37_00.524791/model-00004-1.22580-0.48459-1.11471-0.62500.h5\n",
      "Epoch 5/30\n",
      "21/21 [==============================] - 12s 565ms/step - loss: 1.1148 - categorical_accuracy: 0.5378 - val_loss: 1.2608 - val_categorical_accuracy: 0.4375\n",
      "\n",
      "Epoch 00005: saving model to model_init_2020-12-2713_37_00.524791/model-00005-1.11476-0.53782-1.26076-0.43750.h5\n",
      "Epoch 6/30\n",
      "21/21 [==============================] - 11s 542ms/step - loss: 1.1024 - categorical_accuracy: 0.5770 - val_loss: 1.4099 - val_categorical_accuracy: 0.4375\n",
      "\n",
      "Epoch 00006: saving model to model_init_2020-12-2713_37_00.524791/model-00006-1.10237-0.57703-1.40989-0.43750.h5\n",
      "\n",
      "Epoch 00006: ReduceLROnPlateau reducing learning rate to 0.0005000000237487257.\n",
      "Epoch 7/30\n",
      "21/21 [==============================] - 11s 525ms/step - loss: 0.9711 - categorical_accuracy: 0.6246 - val_loss: 1.6137 - val_categorical_accuracy: 0.1875\n",
      "\n",
      "Epoch 00007: saving model to model_init_2020-12-2713_37_00.524791/model-00007-0.97109-0.62465-1.61374-0.18750.h5\n",
      "Epoch 8/30\n",
      "21/21 [==============================] - 11s 527ms/step - loss: 0.9480 - categorical_accuracy: 0.6246 - val_loss: 0.9862 - val_categorical_accuracy: 0.5625\n",
      "\n",
      "Epoch 00008: saving model to model_init_2020-12-2713_37_00.524791/model-00008-0.94798-0.62465-0.98624-0.56250.h5\n",
      "Epoch 9/30\n",
      "21/21 [==============================] - 11s 540ms/step - loss: 0.9122 - categorical_accuracy: 0.6387 - val_loss: 1.1339 - val_categorical_accuracy: 0.5625\n",
      "\n",
      "Epoch 00009: saving model to model_init_2020-12-2713_37_00.524791/model-00009-0.91217-0.63866-1.13388-0.56250.h5\n",
      "Epoch 10/30\n",
      "21/21 [==============================] - 12s 558ms/step - loss: 0.8390 - categorical_accuracy: 0.6779 - val_loss: 0.9593 - val_categorical_accuracy: 0.6875\n",
      "\n",
      "Epoch 00010: saving model to model_init_2020-12-2713_37_00.524791/model-00010-0.83897-0.67787-0.95928-0.68750.h5\n",
      "Epoch 11/30\n",
      "21/21 [==============================] - 11s 523ms/step - loss: 0.8408 - categorical_accuracy: 0.6863 - val_loss: 1.0119 - val_categorical_accuracy: 0.7500\n",
      "\n",
      "Epoch 00011: saving model to model_init_2020-12-2713_37_00.524791/model-00011-0.84082-0.68627-1.01192-0.75000.h5\n",
      "Epoch 12/30\n",
      "21/21 [==============================] - 12s 552ms/step - loss: 0.7625 - categorical_accuracy: 0.7031 - val_loss: 0.7961 - val_categorical_accuracy: 0.7500\n",
      "\n",
      "Epoch 00012: saving model to model_init_2020-12-2713_37_00.524791/model-00012-0.76252-0.70308-0.79613-0.75000.h5\n",
      "Epoch 13/30\n",
      "21/21 [==============================] - 11s 506ms/step - loss: 0.6996 - categorical_accuracy: 0.7479 - val_loss: 1.0447 - val_categorical_accuracy: 0.8125\n",
      "\n",
      "Epoch 00013: saving model to model_init_2020-12-2713_37_00.524791/model-00013-0.69956-0.74790-1.04473-0.81250.h5\n",
      "Epoch 14/30\n",
      "21/21 [==============================] - 11s 542ms/step - loss: 0.5692 - categorical_accuracy: 0.8067 - val_loss: 0.9172 - val_categorical_accuracy: 0.6250\n",
      "\n",
      "Epoch 00014: saving model to model_init_2020-12-2713_37_00.524791/model-00014-0.56923-0.80672-0.91716-0.62500.h5\n",
      "\n",
      "Epoch 00014: ReduceLROnPlateau reducing learning rate to 0.0002500000118743628.\n",
      "Epoch 15/30\n",
      "21/21 [==============================] - 12s 554ms/step - loss: 0.5967 - categorical_accuracy: 0.8039 - val_loss: 1.1202 - val_categorical_accuracy: 0.6250\n",
      "\n",
      "Epoch 00015: saving model to model_init_2020-12-2713_37_00.524791/model-00015-0.59669-0.80392-1.12016-0.62500.h5\n",
      "Epoch 16/30\n",
      "21/21 [==============================] - 11s 540ms/step - loss: 0.5097 - categorical_accuracy: 0.8207 - val_loss: 0.5400 - val_categorical_accuracy: 0.7500\n",
      "\n",
      "Epoch 00016: saving model to model_init_2020-12-2713_37_00.524791/model-00016-0.50970-0.82073-0.54000-0.75000.h5\n",
      "Epoch 17/30\n",
      "21/21 [==============================] - 11s 526ms/step - loss: 0.4843 - categorical_accuracy: 0.8515 - val_loss: 0.7268 - val_categorical_accuracy: 0.8750\n",
      "\n",
      "Epoch 00017: saving model to model_init_2020-12-2713_37_00.524791/model-00017-0.48426-0.85154-0.72683-0.87500.h5\n",
      "Epoch 18/30\n",
      "21/21 [==============================] - 11s 526ms/step - loss: 0.4644 - categorical_accuracy: 0.8543 - val_loss: 0.7620 - val_categorical_accuracy: 0.7500\n",
      "\n",
      "Epoch 00018: saving model to model_init_2020-12-2713_37_00.524791/model-00018-0.46440-0.85434-0.76205-0.75000.h5\n",
      "\n",
      "Epoch 00018: ReduceLROnPlateau reducing learning rate to 0.0001250000059371814.\n",
      "Epoch 19/30\n",
      "21/21 [==============================] - 11s 532ms/step - loss: 0.4707 - categorical_accuracy: 0.8459 - val_loss: 0.2204 - val_categorical_accuracy: 0.9375\n",
      "\n",
      "Epoch 00019: saving model to model_init_2020-12-2713_37_00.524791/model-00019-0.47066-0.84594-0.22039-0.93750.h5\n",
      "Epoch 20/30\n",
      "21/21 [==============================] - 11s 546ms/step - loss: 0.4076 - categorical_accuracy: 0.8431 - val_loss: 0.7041 - val_categorical_accuracy: 0.6875\n",
      "\n",
      "Epoch 00020: saving model to model_init_2020-12-2713_37_00.524791/model-00020-0.40758-0.84314-0.70412-0.68750.h5\n",
      "Epoch 21/30\n",
      "21/21 [==============================] - 11s 546ms/step - loss: 0.4148 - categorical_accuracy: 0.8515 - val_loss: 0.5445 - val_categorical_accuracy: 0.7500\n",
      "\n",
      "Epoch 00021: saving model to model_init_2020-12-2713_37_00.524791/model-00021-0.41482-0.85154-0.54453-0.75000.h5\n",
      "\n",
      "Epoch 00021: ReduceLROnPlateau reducing learning rate to 6.25000029685907e-05.\n",
      "Epoch 22/30\n",
      "21/21 [==============================] - 11s 534ms/step - loss: 0.3559 - categorical_accuracy: 0.8964 - val_loss: 0.6061 - val_categorical_accuracy: 0.8125\n",
      "\n",
      "Epoch 00022: saving model to model_init_2020-12-2713_37_00.524791/model-00022-0.35594-0.89636-0.60613-0.81250.h5\n",
      "Epoch 23/30\n",
      "21/21 [==============================] - 11s 530ms/step - loss: 0.3532 - categorical_accuracy: 0.9020 - val_loss: 0.6725 - val_categorical_accuracy: 0.8125\n",
      "\n",
      "Epoch 00023: saving model to model_init_2020-12-2713_37_00.524791/model-00023-0.35322-0.90196-0.67252-0.81250.h5\n",
      "\n",
      "Epoch 00023: ReduceLROnPlateau reducing learning rate to 3.125000148429535e-05.\n",
      "Epoch 24/30\n",
      "21/21 [==============================] - 11s 527ms/step - loss: 0.3846 - categorical_accuracy: 0.8796 - val_loss: 0.4095 - val_categorical_accuracy: 0.8750\n",
      "\n",
      "Epoch 00024: saving model to model_init_2020-12-2713_37_00.524791/model-00024-0.38463-0.87955-0.40945-0.87500.h5\n",
      "Epoch 25/30\n",
      "21/21 [==============================] - 11s 523ms/step - loss: 0.3310 - categorical_accuracy: 0.9104 - val_loss: 0.6786 - val_categorical_accuracy: 0.8125\n",
      "\n",
      "Epoch 00025: saving model to model_init_2020-12-2713_37_00.524791/model-00025-0.33104-0.91036-0.67856-0.81250.h5\n",
      "\n",
      "Epoch 00025: ReduceLROnPlateau reducing learning rate to 1.5625000742147677e-05.\n",
      "Epoch 26/30\n",
      "21/21 [==============================] - 11s 531ms/step - loss: 0.3732 - categorical_accuracy: 0.9076 - val_loss: 1.0789 - val_categorical_accuracy: 0.5625\n",
      "\n",
      "Epoch 00026: saving model to model_init_2020-12-2713_37_00.524791/model-00026-0.37318-0.90756-1.07892-0.56250.h5\n",
      "Epoch 27/30\n",
      "21/21 [==============================] - 11s 527ms/step - loss: 0.3133 - categorical_accuracy: 0.9076 - val_loss: 0.8586 - val_categorical_accuracy: 0.6875\n",
      "\n",
      "Epoch 00027: saving model to model_init_2020-12-2713_37_00.524791/model-00027-0.31330-0.90756-0.85861-0.68750.h5\n",
      "\n",
      "Epoch 00027: ReduceLROnPlateau reducing learning rate to 7.812500371073838e-06.\n",
      "Epoch 28/30\n",
      "21/21 [==============================] - 12s 549ms/step - loss: 0.3240 - categorical_accuracy: 0.9076 - val_loss: 0.4932 - val_categorical_accuracy: 0.8125\n",
      "\n",
      "Epoch 00028: saving model to model_init_2020-12-2713_37_00.524791/model-00028-0.32401-0.90756-0.49319-0.81250.h5\n",
      "Epoch 29/30\n",
      "21/21 [==============================] - 11s 529ms/step - loss: 0.3720 - categorical_accuracy: 0.8852 - val_loss: 0.6874 - val_categorical_accuracy: 0.8125\n",
      "\n",
      "Epoch 00029: saving model to model_init_2020-12-2713_37_00.524791/model-00029-0.37204-0.88515-0.68743-0.81250.h5\n",
      "\n",
      "Epoch 00029: ReduceLROnPlateau reducing learning rate to 3.906250185536919e-06.\n",
      "Epoch 30/30\n",
      "21/21 [==============================] - 11s 529ms/step - loss: 0.3122 - categorical_accuracy: 0.9216 - val_loss: 0.2476 - val_categorical_accuracy: 1.0000\n",
      "\n",
      "Epoch 00030: saving model to model_init_2020-12-2713_37_00.524791/model-00030-0.31221-0.92157-0.24757-1.00000.h5\n"
     ]
    }
   ],
   "source": [
    "history9= model9.fit_generator(train_generator, steps_per_epoch=steps_per_epoch, epochs=num_epochs, verbose=1, \n",
    "                    callbacks=callbacks_list, validation_data=val_generator, \n",
    "                    validation_steps=validation_steps, class_weight=None, workers=1, initial_epoch=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Source path =  ./Project_data/val ; batch size = 32\n",
      "Source path =  ./Project_data/trainEpoch 1/30\n",
      " ; batch size = 32\n",
      "21/21 [==============================] - 31s 1s/step - loss: 1.6974 - categorical_accuracy: 0.2397 - val_loss: 1.4495 - val_categorical_accuracy: 0.3900\n",
      "\n",
      "Epoch 00001: saving model to model_init_2020-12-2714_18_00.104298/model-00001-1.70086-0.23831-1.44947-0.39000.h5\n",
      "Epoch 2/30\n",
      "21/21 [==============================] - 12s 584ms/step - loss: 1.3705 - categorical_accuracy: 0.4555 - val_loss: 1.5008 - val_categorical_accuracy: 0.4375\n",
      "\n",
      "Epoch 00002: saving model to model_init_2020-12-2714_18_00.104298/model-00002-1.37049-0.45549-1.50084-0.43750.h5\n",
      "Epoch 3/30\n",
      "21/21 [==============================] - 13s 634ms/step - loss: 1.2873 - categorical_accuracy: 0.4794 - val_loss: 1.5801 - val_categorical_accuracy: 0.4375\n",
      "\n",
      "Epoch 00003: saving model to model_init_2020-12-2714_18_00.104298/model-00003-1.29073-0.47775-1.58008-0.43750.h5\n",
      "\n",
      "Epoch 00003: ReduceLROnPlateau reducing learning rate to 0.0005000000237487257.\n",
      "Epoch 4/30\n",
      "21/21 [==============================] - 11s 516ms/step - loss: 1.2158 - categorical_accuracy: 0.5113 - val_loss: 1.4367 - val_categorical_accuracy: 0.3750\n",
      "\n",
      "Epoch 00004: saving model to model_init_2020-12-2714_18_00.104298/model-00004-1.21579-0.51128-1.43674-0.37500.h5\n",
      "Epoch 5/30\n",
      "21/21 [==============================] - 12s 561ms/step - loss: 1.1717 - categorical_accuracy: 0.5378 - val_loss: 1.1965 - val_categorical_accuracy: 0.5625\n",
      "\n",
      "Epoch 00005: saving model to model_init_2020-12-2714_18_00.104298/model-00005-1.17170-0.53782-1.19651-0.56250.h5\n",
      "Epoch 6/30\n",
      "21/21 [==============================] - 11s 547ms/step - loss: 1.1136 - categorical_accuracy: 0.5686 - val_loss: 1.2002 - val_categorical_accuracy: 0.6250\n",
      "\n",
      "Epoch 00006: saving model to model_init_2020-12-2714_18_00.104298/model-00006-1.11364-0.56863-1.20017-0.62500.h5\n",
      "Epoch 7/30\n",
      "21/21 [==============================] - 11s 533ms/step - loss: 1.1088 - categorical_accuracy: 0.5826 - val_loss: 1.1515 - val_categorical_accuracy: 0.5000\n",
      "\n",
      "Epoch 00007: saving model to model_init_2020-12-2714_18_00.104298/model-00007-1.10880-0.58263-1.15150-0.50000.h5\n",
      "Epoch 8/30\n",
      "21/21 [==============================] - 12s 578ms/step - loss: 1.0207 - categorical_accuracy: 0.5966 - val_loss: 1.2729 - val_categorical_accuracy: 0.5625\n",
      "\n",
      "Epoch 00008: saving model to model_init_2020-12-2714_18_00.104298/model-00008-1.02068-0.59664-1.27290-0.56250.h5\n",
      "Epoch 9/30\n",
      "21/21 [==============================] - 11s 537ms/step - loss: 0.9568 - categorical_accuracy: 0.6387 - val_loss: 0.7913 - val_categorical_accuracy: 0.6875\n",
      "\n",
      "Epoch 00009: saving model to model_init_2020-12-2714_18_00.104298/model-00009-0.95675-0.63866-0.79133-0.68750.h5\n",
      "Epoch 10/30\n",
      "21/21 [==============================] - 12s 573ms/step - loss: 0.8814 - categorical_accuracy: 0.6723 - val_loss: 1.1135 - val_categorical_accuracy: 0.4375\n",
      "\n",
      "Epoch 00010: saving model to model_init_2020-12-2714_18_00.104298/model-00010-0.88141-0.67227-1.11349-0.43750.h5\n",
      "Epoch 11/30\n",
      "21/21 [==============================] - 12s 565ms/step - loss: 1.0094 - categorical_accuracy: 0.5994 - val_loss: 1.2560 - val_categorical_accuracy: 0.4375\n",
      "\n",
      "Epoch 00011: saving model to model_init_2020-12-2714_18_00.104298/model-00011-1.00940-0.59944-1.25600-0.43750.h5\n",
      "\n",
      "Epoch 00011: ReduceLROnPlateau reducing learning rate to 0.0002500000118743628.\n",
      "Epoch 12/30\n",
      "21/21 [==============================] - 11s 546ms/step - loss: 0.8824 - categorical_accuracy: 0.6919 - val_loss: 1.2061 - val_categorical_accuracy: 0.4375\n",
      "\n",
      "Epoch 00012: saving model to model_init_2020-12-2714_18_00.104298/model-00012-0.88242-0.69188-1.20607-0.43750.h5\n",
      "Epoch 13/30\n",
      "21/21 [==============================] - 11s 539ms/step - loss: 0.8195 - categorical_accuracy: 0.6835 - val_loss: 0.6980 - val_categorical_accuracy: 0.6875\n",
      "\n",
      "Epoch 00013: saving model to model_init_2020-12-2714_18_00.104298/model-00013-0.81950-0.68347-0.69802-0.68750.h5\n",
      "Epoch 14/30\n",
      "21/21 [==============================] - 12s 570ms/step - loss: 0.7110 - categorical_accuracy: 0.7367 - val_loss: 0.6901 - val_categorical_accuracy: 0.7500\n",
      "\n",
      "Epoch 00014: saving model to model_init_2020-12-2714_18_00.104298/model-00014-0.71102-0.73669-0.69008-0.75000.h5\n",
      "Epoch 15/30\n",
      "21/21 [==============================] - 11s 547ms/step - loss: 0.6841 - categorical_accuracy: 0.7451 - val_loss: 0.7055 - val_categorical_accuracy: 0.7500\n",
      "\n",
      "Epoch 00015: saving model to model_init_2020-12-2714_18_00.104298/model-00015-0.68407-0.74510-0.70549-0.75000.h5\n",
      "Epoch 16/30\n",
      "21/21 [==============================] - 11s 541ms/step - loss: 0.6825 - categorical_accuracy: 0.7647 - val_loss: 0.7428 - val_categorical_accuracy: 0.5625\n",
      "\n",
      "Epoch 00016: saving model to model_init_2020-12-2714_18_00.104298/model-00016-0.68249-0.76471-0.74278-0.56250.h5\n",
      "\n",
      "Epoch 00016: ReduceLROnPlateau reducing learning rate to 0.0001250000059371814.\n",
      "Epoch 17/30\n",
      "21/21 [==============================] - 12s 561ms/step - loss: 0.6289 - categorical_accuracy: 0.7955 - val_loss: 0.8076 - val_categorical_accuracy: 0.6250\n",
      "\n",
      "Epoch 00017: saving model to model_init_2020-12-2714_18_00.104298/model-00017-0.62892-0.79552-0.80757-0.62500.h5\n",
      "Epoch 18/30\n",
      "21/21 [==============================] - 12s 562ms/step - loss: 0.5849 - categorical_accuracy: 0.8095 - val_loss: 0.7301 - val_categorical_accuracy: 0.6250\n",
      "\n",
      "Epoch 00018: saving model to model_init_2020-12-2714_18_00.104298/model-00018-0.58493-0.80952-0.73007-0.62500.h5\n",
      "\n",
      "Epoch 00018: ReduceLROnPlateau reducing learning rate to 6.25000029685907e-05.\n",
      "Epoch 19/30\n",
      "21/21 [==============================] - 11s 540ms/step - loss: 0.5595 - categorical_accuracy: 0.8151 - val_loss: 0.6035 - val_categorical_accuracy: 0.6875\n",
      "\n",
      "Epoch 00019: saving model to model_init_2020-12-2714_18_00.104298/model-00019-0.55952-0.81513-0.60353-0.68750.h5\n",
      "Epoch 20/30\n",
      "21/21 [==============================] - 12s 562ms/step - loss: 0.5489 - categorical_accuracy: 0.8039 - val_loss: 0.8932 - val_categorical_accuracy: 0.5625\n",
      "\n",
      "Epoch 00020: saving model to model_init_2020-12-2714_18_00.104298/model-00020-0.54889-0.80392-0.89319-0.56250.h5\n",
      "Epoch 21/30\n",
      "21/21 [==============================] - 11s 525ms/step - loss: 0.5291 - categorical_accuracy: 0.8347 - val_loss: 0.5163 - val_categorical_accuracy: 0.8125\n",
      "\n",
      "Epoch 00021: saving model to model_init_2020-12-2714_18_00.104298/model-00021-0.52908-0.83473-0.51632-0.81250.h5\n",
      "Epoch 22/30\n",
      "21/21 [==============================] - 12s 567ms/step - loss: 0.5460 - categorical_accuracy: 0.8235 - val_loss: 0.6557 - val_categorical_accuracy: 0.7500\n",
      "\n",
      "Epoch 00022: saving model to model_init_2020-12-2714_18_00.104298/model-00022-0.54604-0.82353-0.65566-0.75000.h5\n",
      "Epoch 23/30\n",
      "21/21 [==============================] - 11s 547ms/step - loss: 0.5116 - categorical_accuracy: 0.8319 - val_loss: 0.6609 - val_categorical_accuracy: 0.6875\n",
      "\n",
      "Epoch 00023: saving model to model_init_2020-12-2714_18_00.104298/model-00023-0.51164-0.83193-0.66094-0.68750.h5\n",
      "\n",
      "Epoch 00023: ReduceLROnPlateau reducing learning rate to 3.125000148429535e-05.\n",
      "Epoch 24/30\n",
      "21/21 [==============================] - 11s 545ms/step - loss: 0.5236 - categorical_accuracy: 0.8291 - val_loss: 0.7500 - val_categorical_accuracy: 0.7500\n",
      "\n",
      "Epoch 00024: saving model to model_init_2020-12-2714_18_00.104298/model-00024-0.52359-0.82913-0.74999-0.75000.h5\n",
      "Epoch 25/30\n",
      "21/21 [==============================] - 12s 554ms/step - loss: 0.5183 - categorical_accuracy: 0.8235 - val_loss: 0.7503 - val_categorical_accuracy: 0.6875\n",
      "\n",
      "Epoch 00025: saving model to model_init_2020-12-2714_18_00.104298/model-00025-0.51833-0.82353-0.75031-0.68750.h5\n",
      "\n",
      "Epoch 00025: ReduceLROnPlateau reducing learning rate to 1.5625000742147677e-05.\n",
      "Epoch 26/30\n",
      "21/21 [==============================] - 11s 543ms/step - loss: 0.4395 - categorical_accuracy: 0.8739 - val_loss: 1.0686 - val_categorical_accuracy: 0.5000\n",
      "\n",
      "Epoch 00026: saving model to model_init_2020-12-2714_18_00.104298/model-00026-0.43950-0.87395-1.06864-0.50000.h5\n",
      "Epoch 27/30\n",
      "21/21 [==============================] - 11s 541ms/step - loss: 0.4477 - categorical_accuracy: 0.8459 - val_loss: 0.4973 - val_categorical_accuracy: 0.7500\n",
      "\n",
      "Epoch 00027: saving model to model_init_2020-12-2714_18_00.104298/model-00027-0.44773-0.84594-0.49733-0.75000.h5\n",
      "Epoch 28/30\n",
      "21/21 [==============================] - 12s 564ms/step - loss: 0.4850 - categorical_accuracy: 0.8431 - val_loss: 0.7928 - val_categorical_accuracy: 0.6250\n",
      "\n",
      "Epoch 00028: saving model to model_init_2020-12-2714_18_00.104298/model-00028-0.48496-0.84314-0.79279-0.62500.h5\n",
      "Epoch 29/30\n",
      "21/21 [==============================] - 12s 573ms/step - loss: 0.4721 - categorical_accuracy: 0.8487 - val_loss: 0.7499 - val_categorical_accuracy: 0.6875\n",
      "\n",
      "Epoch 00029: saving model to model_init_2020-12-2714_18_00.104298/model-00029-0.47212-0.84874-0.74991-0.68750.h5\n",
      "\n",
      "Epoch 00029: ReduceLROnPlateau reducing learning rate to 7.812500371073838e-06.\n",
      "Epoch 30/30\n",
      "21/21 [==============================] - 11s 543ms/step - loss: 0.4485 - categorical_accuracy: 0.8571 - val_loss: 0.8633 - val_categorical_accuracy: 0.6875\n",
      "\n",
      "Epoch 00030: saving model to model_init_2020-12-2714_18_00.104298/model-00030-0.44851-0.85714-0.86333-0.68750.h5\n"
     ]
    }
   ],
   "source": [
    "history10= model10.fit_generator(train_generator, steps_per_epoch=steps_per_epoch, epochs=num_epochs, verbose=1, \n",
    "                    callbacks=callbacks_list, validation_data=val_generator, \n",
    "                    validation_steps=validation_steps, class_weight=None, workers=1, initial_epoch=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/30\n",
      "21/21 [==============================] - 12s 580ms/step - loss: 1.7037 - categorical_accuracy: 0.3025 - val_loss: 1.4066 - val_categorical_accuracy: 0.4375\n",
      "\n",
      "Epoch 00001: saving model to model_init_2020-12-2714_18_00.104298/model-00001-1.70372-0.30252-1.40665-0.43750.h5\n",
      "Epoch 2/30\n",
      "21/21 [==============================] - 10s 498ms/step - loss: 1.4660 - categorical_accuracy: 0.3754 - val_loss: 1.6307 - val_categorical_accuracy: 0.2500\n",
      "\n",
      "Epoch 00002: saving model to model_init_2020-12-2714_18_00.104298/model-00002-1.46597-0.37535-1.63074-0.25000.h5\n",
      "Epoch 3/30\n",
      "21/21 [==============================] - 11s 539ms/step - loss: 1.3914 - categorical_accuracy: 0.3950 - val_loss: 1.1860 - val_categorical_accuracy: 0.5625\n",
      "\n",
      "Epoch 00003: saving model to model_init_2020-12-2714_18_00.104298/model-00003-1.39141-0.39496-1.18595-0.56250.h5\n",
      "Epoch 4/30\n",
      "21/21 [==============================] - 11s 528ms/step - loss: 1.2671 - categorical_accuracy: 0.4314 - val_loss: 1.7259 - val_categorical_accuracy: 0.2500\n",
      "\n",
      "Epoch 00004: saving model to model_init_2020-12-2714_18_00.104298/model-00004-1.26715-0.43137-1.72591-0.25000.h5\n",
      "Epoch 5/30\n",
      "21/21 [==============================] - 11s 528ms/step - loss: 1.1950 - categorical_accuracy: 0.5070 - val_loss: 1.3633 - val_categorical_accuracy: 0.5000\n",
      "\n",
      "Epoch 00005: saving model to model_init_2020-12-2714_18_00.104298/model-00005-1.19502-0.50700-1.36333-0.50000.h5\n",
      "\n",
      "Epoch 00005: ReduceLROnPlateau reducing learning rate to 0.0005000000237487257.\n",
      "Epoch 6/30\n",
      "21/21 [==============================] - 12s 566ms/step - loss: 1.0853 - categorical_accuracy: 0.5686 - val_loss: 1.0663 - val_categorical_accuracy: 0.5000\n",
      "\n",
      "Epoch 00006: saving model to model_init_2020-12-2714_18_00.104298/model-00006-1.08532-0.56863-1.06634-0.50000.h5\n",
      "Epoch 7/30\n",
      "21/21 [==============================] - 12s 551ms/step - loss: 0.9981 - categorical_accuracy: 0.6022 - val_loss: 0.8485 - val_categorical_accuracy: 0.7500\n",
      "\n",
      "Epoch 00007: saving model to model_init_2020-12-2714_18_00.104298/model-00007-0.99806-0.60224-0.84852-0.75000.h5\n",
      "Epoch 8/30\n",
      "21/21 [==============================] - 12s 549ms/step - loss: 1.0552 - categorical_accuracy: 0.5462 - val_loss: 1.1471 - val_categorical_accuracy: 0.5625\n",
      "\n",
      "Epoch 00008: saving model to model_init_2020-12-2714_18_00.104298/model-00008-1.05521-0.54622-1.14705-0.56250.h5\n",
      "Epoch 9/30\n",
      "21/21 [==============================] - 11s 515ms/step - loss: 0.9309 - categorical_accuracy: 0.6723 - val_loss: 1.3551 - val_categorical_accuracy: 0.5000\n",
      "\n",
      "Epoch 00009: saving model to model_init_2020-12-2714_18_00.104298/model-00009-0.93090-0.67227-1.35512-0.50000.h5\n",
      "\n",
      "Epoch 00009: ReduceLROnPlateau reducing learning rate to 0.0002500000118743628.\n",
      "Epoch 10/30\n",
      "21/21 [==============================] - 11s 512ms/step - loss: 0.9199 - categorical_accuracy: 0.6415 - val_loss: 0.8755 - val_categorical_accuracy: 0.6250\n",
      "\n",
      "Epoch 00010: saving model to model_init_2020-12-2714_18_00.104298/model-00010-0.91989-0.64146-0.87555-0.62500.h5\n",
      "Epoch 11/30\n",
      "21/21 [==============================] - 12s 570ms/step - loss: 0.8502 - categorical_accuracy: 0.6695 - val_loss: 0.7146 - val_categorical_accuracy: 0.6875\n",
      "\n",
      "Epoch 00011: saving model to model_init_2020-12-2714_18_00.104298/model-00011-0.85025-0.66947-0.71460-0.68750.h5\n",
      "Epoch 12/30\n",
      "21/21 [==============================] - 12s 568ms/step - loss: 0.8101 - categorical_accuracy: 0.6863 - val_loss: 0.9446 - val_categorical_accuracy: 0.6875\n",
      "\n",
      "Epoch 00012: saving model to model_init_2020-12-2714_18_00.104298/model-00012-0.81013-0.68627-0.94464-0.68750.h5\n",
      "Epoch 13/30\n",
      "21/21 [==============================] - 11s 506ms/step - loss: 0.7733 - categorical_accuracy: 0.7311 - val_loss: 0.6216 - val_categorical_accuracy: 0.6875\n",
      "\n",
      "Epoch 00013: saving model to model_init_2020-12-2714_18_00.104298/model-00013-0.77325-0.73109-0.62161-0.68750.h5\n",
      "Epoch 14/30\n",
      "21/21 [==============================] - 11s 534ms/step - loss: 0.7830 - categorical_accuracy: 0.6947 - val_loss: 0.7364 - val_categorical_accuracy: 0.6875\n",
      "\n",
      "Epoch 00014: saving model to model_init_2020-12-2714_18_00.104298/model-00014-0.78298-0.69468-0.73641-0.68750.h5\n",
      "Epoch 15/30\n",
      "21/21 [==============================] - 12s 550ms/step - loss: 0.7699 - categorical_accuracy: 0.7451 - val_loss: 0.7409 - val_categorical_accuracy: 0.8125\n",
      "\n",
      "Epoch 00015: saving model to model_init_2020-12-2714_18_00.104298/model-00015-0.76992-0.74510-0.74090-0.81250.h5\n",
      "\n",
      "Epoch 00015: ReduceLROnPlateau reducing learning rate to 0.0001250000059371814.\n",
      "Epoch 16/30\n",
      "21/21 [==============================] - 12s 562ms/step - loss: 0.6482 - categorical_accuracy: 0.7367 - val_loss: 0.6526 - val_categorical_accuracy: 0.7500\n",
      "\n",
      "Epoch 00016: saving model to model_init_2020-12-2714_18_00.104298/model-00016-0.64818-0.73669-0.65264-0.75000.h5\n",
      "Epoch 17/30\n",
      "21/21 [==============================] - 11s 517ms/step - loss: 0.5706 - categorical_accuracy: 0.7927 - val_loss: 0.5878 - val_categorical_accuracy: 0.8125\n",
      "\n",
      "Epoch 00017: saving model to model_init_2020-12-2714_18_00.104298/model-00017-0.57059-0.79272-0.58779-0.81250.h5\n",
      "Epoch 18/30\n",
      "21/21 [==============================] - 11s 534ms/step - loss: 0.6177 - categorical_accuracy: 0.7815 - val_loss: 1.2707 - val_categorical_accuracy: 0.6250\n",
      "\n",
      "Epoch 00018: saving model to model_init_2020-12-2714_18_00.104298/model-00018-0.61768-0.78151-1.27073-0.62500.h5\n",
      "Epoch 19/30\n",
      "21/21 [==============================] - 12s 572ms/step - loss: 0.5701 - categorical_accuracy: 0.7927 - val_loss: 0.5947 - val_categorical_accuracy: 0.8125\n",
      "\n",
      "Epoch 00019: saving model to model_init_2020-12-2714_18_00.104298/model-00019-0.57015-0.79272-0.59472-0.81250.h5\n",
      "\n",
      "Epoch 00019: ReduceLROnPlateau reducing learning rate to 6.25000029685907e-05.\n",
      "Epoch 20/30\n",
      "21/21 [==============================] - 11s 524ms/step - loss: 0.6241 - categorical_accuracy: 0.7955 - val_loss: 0.9530 - val_categorical_accuracy: 0.5625\n",
      "\n",
      "Epoch 00020: saving model to model_init_2020-12-2714_18_00.104298/model-00020-0.62414-0.79552-0.95299-0.56250.h5\n",
      "Epoch 21/30\n",
      "21/21 [==============================] - 12s 551ms/step - loss: 0.6221 - categorical_accuracy: 0.7675 - val_loss: 0.7629 - val_categorical_accuracy: 0.6875\n",
      "\n",
      "Epoch 00021: saving model to model_init_2020-12-2714_18_00.104298/model-00021-0.62208-0.76751-0.76286-0.68750.h5\n",
      "\n",
      "Epoch 00021: ReduceLROnPlateau reducing learning rate to 3.125000148429535e-05.\n",
      "Epoch 22/30\n",
      "21/21 [==============================] - 12s 549ms/step - loss: 0.5525 - categorical_accuracy: 0.8151 - val_loss: 0.4934 - val_categorical_accuracy: 0.7500\n",
      "\n",
      "Epoch 00022: saving model to model_init_2020-12-2714_18_00.104298/model-00022-0.55252-0.81513-0.49342-0.75000.h5\n",
      "Epoch 23/30\n",
      "21/21 [==============================] - 11s 539ms/step - loss: 0.5756 - categorical_accuracy: 0.7927 - val_loss: 0.5926 - val_categorical_accuracy: 0.7500\n",
      "\n",
      "Epoch 00023: saving model to model_init_2020-12-2714_18_00.104298/model-00023-0.57560-0.79272-0.59264-0.75000.h5\n",
      "Epoch 24/30\n",
      "21/21 [==============================] - 11s 525ms/step - loss: 0.4754 - categorical_accuracy: 0.8515 - val_loss: 0.4012 - val_categorical_accuracy: 0.8125\n",
      "\n",
      "Epoch 00024: saving model to model_init_2020-12-2714_18_00.104298/model-00024-0.47538-0.85154-0.40118-0.81250.h5\n",
      "Epoch 25/30\n",
      "21/21 [==============================] - 11s 535ms/step - loss: 0.5492 - categorical_accuracy: 0.8235 - val_loss: 0.4933 - val_categorical_accuracy: 0.8750\n",
      "\n",
      "Epoch 00025: saving model to model_init_2020-12-2714_18_00.104298/model-00025-0.54924-0.82353-0.49328-0.87500.h5\n",
      "Epoch 26/30\n",
      "21/21 [==============================] - 12s 550ms/step - loss: 0.5402 - categorical_accuracy: 0.8403 - val_loss: 0.8107 - val_categorical_accuracy: 0.6250\n",
      "\n",
      "Epoch 00026: saving model to model_init_2020-12-2714_18_00.104298/model-00026-0.54024-0.84034-0.81071-0.62500.h5\n",
      "\n",
      "Epoch 00026: ReduceLROnPlateau reducing learning rate to 1.5625000742147677e-05.\n",
      "Epoch 27/30\n",
      "21/21 [==============================] - 11s 533ms/step - loss: 0.5020 - categorical_accuracy: 0.8319 - val_loss: 0.5107 - val_categorical_accuracy: 0.7500\n",
      "\n",
      "Epoch 00027: saving model to model_init_2020-12-2714_18_00.104298/model-00027-0.50201-0.83193-0.51073-0.75000.h5\n",
      "Epoch 28/30\n",
      "21/21 [==============================] - 11s 542ms/step - loss: 0.5191 - categorical_accuracy: 0.8235 - val_loss: 0.8810 - val_categorical_accuracy: 0.6875\n",
      "\n",
      "Epoch 00028: saving model to model_init_2020-12-2714_18_00.104298/model-00028-0.51910-0.82353-0.88104-0.68750.h5\n",
      "\n",
      "Epoch 00028: ReduceLROnPlateau reducing learning rate to 7.812500371073838e-06.\n",
      "Epoch 29/30\n",
      "21/21 [==============================] - 12s 572ms/step - loss: 0.5593 - categorical_accuracy: 0.8095 - val_loss: 0.6548 - val_categorical_accuracy: 0.6875\n",
      "\n",
      "Epoch 00029: saving model to model_init_2020-12-2714_18_00.104298/model-00029-0.55928-0.80952-0.65482-0.68750.h5\n",
      "Epoch 30/30\n",
      "21/21 [==============================] - 11s 544ms/step - loss: 0.5560 - categorical_accuracy: 0.8151 - val_loss: 0.7367 - val_categorical_accuracy: 0.7500\n",
      "\n",
      "Epoch 00030: saving model to model_init_2020-12-2714_18_00.104298/model-00030-0.55600-0.81513-0.73667-0.75000.h5\n",
      "\n",
      "Epoch 00030: ReduceLROnPlateau reducing learning rate to 3.906250185536919e-06.\n"
     ]
    }
   ],
   "source": [
    "history11= model11.fit_generator(train_generator, steps_per_epoch=steps_per_epoch, epochs=num_epochs, verbose=1, \n",
    "                    callbacks=callbacks_list, validation_data=val_generator, \n",
    "                    validation_steps=validation_steps, class_weight=None, workers=1, initial_epoch=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Source path =  ./Project_data/val ; batch size = 32\n",
      "Source path =  ./Project_data/train ; batch size = 32Epoch 1/30\n",
      "\n",
      "21/21 [==============================] - 26s 1s/step - loss: 1.6475 - categorical_accuracy: 0.2446 - val_loss: 1.6330 - val_categorical_accuracy: 0.2400\n",
      "\n",
      "Epoch 00001: saving model to model_init_2020-12-2715_29_07.909081/model-00001-1.64979-0.24434-1.63301-0.24000.h5\n",
      "Epoch 2/30\n",
      "21/21 [==============================] - 13s 601ms/step - loss: 1.5070 - categorical_accuracy: 0.3499 - val_loss: 1.4850 - val_categorical_accuracy: 0.4375\n",
      "\n",
      "Epoch 00002: saving model to model_init_2020-12-2715_29_07.909081/model-00002-1.50700-0.34990-1.48504-0.43750.h5\n",
      "Epoch 3/30\n",
      "21/21 [==============================] - 13s 638ms/step - loss: 1.4351 - categorical_accuracy: 0.3878 - val_loss: 1.9895 - val_categorical_accuracy: 0.0625\n",
      "\n",
      "Epoch 00003: saving model to model_init_2020-12-2715_29_07.909081/model-00003-1.43833-0.38642-1.98947-0.06250.h5\n",
      "Epoch 4/30\n",
      "21/21 [==============================] - 12s 585ms/step - loss: 1.3927 - categorical_accuracy: 0.4135 - val_loss: 1.5077 - val_categorical_accuracy: 0.3125\n",
      "\n",
      "Epoch 00004: saving model to model_init_2020-12-2715_29_07.909081/model-00004-1.39268-0.41353-1.50774-0.31250.h5\n",
      "\n",
      "Epoch 00004: ReduceLROnPlateau reducing learning rate to 0.0005000000237487257.\n",
      "Epoch 5/30\n",
      "21/21 [==============================] - 11s 529ms/step - loss: 1.2653 - categorical_accuracy: 0.5070 - val_loss: 1.6923 - val_categorical_accuracy: 0.1875\n",
      "\n",
      "Epoch 00005: saving model to model_init_2020-12-2715_29_07.909081/model-00005-1.26531-0.50700-1.69233-0.18750.h5\n",
      "Epoch 6/30\n",
      "21/21 [==============================] - 11s 541ms/step - loss: 1.2397 - categorical_accuracy: 0.5406 - val_loss: 1.4356 - val_categorical_accuracy: 0.3125\n",
      "\n",
      "Epoch 00006: saving model to model_init_2020-12-2715_29_07.909081/model-00006-1.23969-0.54062-1.43564-0.31250.h5\n",
      "Epoch 7/30\n",
      "21/21 [==============================] - 11s 538ms/step - loss: 1.1231 - categorical_accuracy: 0.5798 - val_loss: 1.3366 - val_categorical_accuracy: 0.5000\n",
      "\n",
      "Epoch 00007: saving model to model_init_2020-12-2715_29_07.909081/model-00007-1.12311-0.57983-1.33663-0.50000.h5\n",
      "Epoch 8/30\n",
      "21/21 [==============================] - 11s 533ms/step - loss: 1.1137 - categorical_accuracy: 0.5658 - val_loss: 1.6903 - val_categorical_accuracy: 0.3125\n",
      "\n",
      "Epoch 00008: saving model to model_init_2020-12-2715_29_07.909081/model-00008-1.11375-0.56583-1.69034-0.31250.h5\n",
      "Epoch 9/30\n",
      "21/21 [==============================] - 12s 548ms/step - loss: 1.0408 - categorical_accuracy: 0.5882 - val_loss: 1.3671 - val_categorical_accuracy: 0.5625\n",
      "\n",
      "Epoch 00009: saving model to model_init_2020-12-2715_29_07.909081/model-00009-1.04083-0.58824-1.36714-0.56250.h5\n",
      "\n",
      "Epoch 00009: ReduceLROnPlateau reducing learning rate to 0.0002500000118743628.\n",
      "Epoch 10/30\n",
      "21/21 [==============================] - 12s 561ms/step - loss: 0.9949 - categorical_accuracy: 0.6078 - val_loss: 1.3560 - val_categorical_accuracy: 0.5625\n",
      "\n",
      "Epoch 00010: saving model to model_init_2020-12-2715_29_07.909081/model-00010-0.99486-0.60784-1.35599-0.56250.h5\n",
      "Epoch 11/30\n",
      "21/21 [==============================] - 12s 548ms/step - loss: 1.0698 - categorical_accuracy: 0.5798 - val_loss: 1.7405 - val_categorical_accuracy: 0.3750\n",
      "\n",
      "Epoch 00011: saving model to model_init_2020-12-2715_29_07.909081/model-00011-1.06981-0.57983-1.74047-0.37500.h5\n",
      "\n",
      "Epoch 00011: ReduceLROnPlateau reducing learning rate to 0.0001250000059371814.\n",
      "Epoch 12/30\n",
      "21/21 [==============================] - 11s 546ms/step - loss: 1.0331 - categorical_accuracy: 0.6134 - val_loss: 1.8681 - val_categorical_accuracy: 0.2500\n",
      "\n",
      "Epoch 00012: saving model to model_init_2020-12-2715_29_07.909081/model-00012-1.03310-0.61345-1.86813-0.25000.h5\n",
      "Epoch 13/30\n",
      "21/21 [==============================] - 12s 580ms/step - loss: 0.9347 - categorical_accuracy: 0.6471 - val_loss: 1.3775 - val_categorical_accuracy: 0.3750\n",
      "\n",
      "Epoch 00013: saving model to model_init_2020-12-2715_29_07.909081/model-00013-0.93468-0.64706-1.37746-0.37500.h5\n",
      "\n",
      "Epoch 00013: ReduceLROnPlateau reducing learning rate to 6.25000029685907e-05.\n",
      "Epoch 14/30\n",
      "21/21 [==============================] - 11s 502ms/step - loss: 0.9981 - categorical_accuracy: 0.6078 - val_loss: 1.8874 - val_categorical_accuracy: 0.3750\n",
      "\n",
      "Epoch 00014: saving model to model_init_2020-12-2715_29_07.909081/model-00014-0.99811-0.60784-1.88739-0.37500.h5\n",
      "Epoch 15/30\n",
      "21/21 [==============================] - 12s 572ms/step - loss: 0.9522 - categorical_accuracy: 0.6387 - val_loss: 1.6709 - val_categorical_accuracy: 0.5000\n",
      "\n",
      "Epoch 00015: saving model to model_init_2020-12-2715_29_07.909081/model-00015-0.95217-0.63866-1.67094-0.50000.h5\n",
      "\n",
      "Epoch 00015: ReduceLROnPlateau reducing learning rate to 3.125000148429535e-05.\n",
      "Epoch 16/30\n",
      "21/21 [==============================] - 11s 545ms/step - loss: 0.9737 - categorical_accuracy: 0.6387 - val_loss: 1.6690 - val_categorical_accuracy: 0.3125\n",
      "\n",
      "Epoch 00016: saving model to model_init_2020-12-2715_29_07.909081/model-00016-0.97375-0.63866-1.66900-0.31250.h5\n",
      "Epoch 17/30\n",
      "21/21 [==============================] - 12s 565ms/step - loss: 0.9591 - categorical_accuracy: 0.6331 - val_loss: 1.3602 - val_categorical_accuracy: 0.5625\n",
      "\n",
      "Epoch 00017: saving model to model_init_2020-12-2715_29_07.909081/model-00017-0.95911-0.63305-1.36023-0.56250.h5\n",
      "\n",
      "Epoch 00017: ReduceLROnPlateau reducing learning rate to 1.5625000742147677e-05.\n",
      "Epoch 18/30\n",
      "21/21 [==============================] - 12s 572ms/step - loss: 0.9728 - categorical_accuracy: 0.6246 - val_loss: 1.8439 - val_categorical_accuracy: 0.2500\n",
      "\n",
      "Epoch 00018: saving model to model_init_2020-12-2715_29_07.909081/model-00018-0.97279-0.62465-1.84392-0.25000.h5\n",
      "Epoch 19/30\n",
      "21/21 [==============================] - 12s 551ms/step - loss: 0.9809 - categorical_accuracy: 0.6106 - val_loss: 1.5905 - val_categorical_accuracy: 0.2500\n",
      "\n",
      "Epoch 00019: saving model to model_init_2020-12-2715_29_07.909081/model-00019-0.98092-0.61064-1.59046-0.25000.h5\n",
      "\n",
      "Epoch 00019: ReduceLROnPlateau reducing learning rate to 7.812500371073838e-06.\n",
      "Epoch 20/30\n",
      "21/21 [==============================] - 12s 567ms/step - loss: 0.9664 - categorical_accuracy: 0.6303 - val_loss: 1.5846 - val_categorical_accuracy: 0.5000\n",
      "\n",
      "Epoch 00020: saving model to model_init_2020-12-2715_29_07.909081/model-00020-0.96636-0.63025-1.58464-0.50000.h5\n",
      "Epoch 21/30\n",
      "21/21 [==============================] - 12s 559ms/step - loss: 0.9291 - categorical_accuracy: 0.6723 - val_loss: 1.4585 - val_categorical_accuracy: 0.5625\n",
      "\n",
      "Epoch 00021: saving model to model_init_2020-12-2715_29_07.909081/model-00021-0.92914-0.67227-1.45850-0.56250.h5\n",
      "\n",
      "Epoch 00021: ReduceLROnPlateau reducing learning rate to 3.906250185536919e-06.\n",
      "Epoch 22/30\n",
      "21/21 [==============================] - 12s 551ms/step - loss: 0.9915 - categorical_accuracy: 0.6190 - val_loss: 1.5727 - val_categorical_accuracy: 0.3750\n",
      "\n",
      "Epoch 00022: saving model to model_init_2020-12-2715_29_07.909081/model-00022-0.99147-0.61905-1.57274-0.37500.h5\n",
      "Epoch 23/30\n",
      "21/21 [==============================] - 12s 553ms/step - loss: 0.9858 - categorical_accuracy: 0.6639 - val_loss: 1.9995 - val_categorical_accuracy: 0.3125\n",
      "\n",
      "Epoch 00023: saving model to model_init_2020-12-2715_29_07.909081/model-00023-0.98584-0.66387-1.99949-0.31250.h5\n",
      "\n",
      "Epoch 00023: ReduceLROnPlateau reducing learning rate to 1.9531250927684596e-06.\n",
      "Epoch 24/30\n",
      "21/21 [==============================] - 12s 575ms/step - loss: 0.9308 - categorical_accuracy: 0.6555 - val_loss: 1.4440 - val_categorical_accuracy: 0.4375\n",
      "\n",
      "Epoch 00024: saving model to model_init_2020-12-2715_29_07.909081/model-00024-0.93077-0.65546-1.44397-0.43750.h5\n",
      "Epoch 25/30\n",
      "21/21 [==============================] - 12s 552ms/step - loss: 0.9583 - categorical_accuracy: 0.6218 - val_loss: 1.4872 - val_categorical_accuracy: 0.4375\n",
      "\n",
      "Epoch 00025: saving model to model_init_2020-12-2715_29_07.909081/model-00025-0.95832-0.62185-1.48715-0.43750.h5\n",
      "\n",
      "Epoch 00025: ReduceLROnPlateau reducing learning rate to 9.765625463842298e-07.\n",
      "Epoch 26/30\n",
      "21/21 [==============================] - 12s 574ms/step - loss: 0.9865 - categorical_accuracy: 0.6162 - val_loss: 1.5713 - val_categorical_accuracy: 0.3750\n",
      "\n",
      "Epoch 00026: saving model to model_init_2020-12-2715_29_07.909081/model-00026-0.98655-0.61625-1.57132-0.37500.h5\n",
      "Epoch 27/30\n",
      "21/21 [==============================] - 11s 531ms/step - loss: 0.9604 - categorical_accuracy: 0.6246 - val_loss: 1.6797 - val_categorical_accuracy: 0.3125\n",
      "\n",
      "Epoch 00027: saving model to model_init_2020-12-2715_29_07.909081/model-00027-0.96036-0.62465-1.67972-0.31250.h5\n",
      "\n",
      "Epoch 00027: ReduceLROnPlateau reducing learning rate to 4.882812731921149e-07.\n",
      "Epoch 28/30\n",
      "21/21 [==============================] - 12s 580ms/step - loss: 0.9201 - categorical_accuracy: 0.6443 - val_loss: 1.6446 - val_categorical_accuracy: 0.3750\n",
      "\n",
      "Epoch 00028: saving model to model_init_2020-12-2715_29_07.909081/model-00028-0.92015-0.64426-1.64461-0.37500.h5\n",
      "Epoch 29/30\n",
      "21/21 [==============================] - 12s 583ms/step - loss: 1.0522 - categorical_accuracy: 0.5826 - val_loss: 2.0483 - val_categorical_accuracy: 0.4375\n",
      "\n",
      "Epoch 00029: saving model to model_init_2020-12-2715_29_07.909081/model-00029-1.05216-0.58263-2.04827-0.43750.h5\n",
      "\n",
      "Epoch 00029: ReduceLROnPlateau reducing learning rate to 2.4414063659605745e-07.\n",
      "Epoch 30/30\n",
      "21/21 [==============================] - 11s 526ms/step - loss: 0.8605 - categorical_accuracy: 0.6583 - val_loss: 1.9681 - val_categorical_accuracy: 0.2500\n",
      "\n",
      "Epoch 00030: saving model to model_init_2020-12-2715_29_07.909081/model-00030-0.86048-0.65826-1.96811-0.25000.h5\n"
     ]
    }
   ],
   "source": [
    "history12= model12.fit_generator(train_generator, steps_per_epoch=steps_per_epoch, epochs=num_epochs, verbose=1, \n",
    "                    callbacks=callbacks_list, validation_data=val_generator, \n",
    "                    validation_steps=validation_steps, class_weight=None, workers=1, initial_epoch=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/30\n",
      "21/21 [==============================] - 13s 621ms/step - loss: 1.8672 - categorical_accuracy: 0.2661 - val_loss: 1.4052 - val_categorical_accuracy: 0.3125\n",
      "\n",
      "Epoch 00001: saving model to model_init_2020-12-2715_29_07.909081/model-00001-1.86716-0.26611-1.40522-0.31250.h5\n",
      "Epoch 2/30\n",
      "21/21 [==============================] - 11s 502ms/step - loss: 1.6232 - categorical_accuracy: 0.3137 - val_loss: 1.4021 - val_categorical_accuracy: 0.3125\n",
      "\n",
      "Epoch 00002: saving model to model_init_2020-12-2715_29_07.909081/model-00002-1.62315-0.31373-1.40205-0.31250.h5\n",
      "Epoch 3/30\n",
      "21/21 [==============================] - 12s 560ms/step - loss: 1.4973 - categorical_accuracy: 0.3501 - val_loss: 1.5140 - val_categorical_accuracy: 0.2500\n",
      "\n",
      "Epoch 00003: saving model to model_init_2020-12-2715_29_07.909081/model-00003-1.49733-0.35014-1.51397-0.25000.h5\n",
      "Epoch 4/30\n",
      "21/21 [==============================] - 12s 563ms/step - loss: 1.4902 - categorical_accuracy: 0.3613 - val_loss: 1.7696 - val_categorical_accuracy: 0.3750\n",
      "\n",
      "Epoch 00004: saving model to model_init_2020-12-2715_29_07.909081/model-00004-1.49020-0.36134-1.76961-0.37500.h5\n",
      "\n",
      "Epoch 00004: ReduceLROnPlateau reducing learning rate to 0.0005000000237487257.\n",
      "Epoch 5/30\n",
      "21/21 [==============================] - 12s 568ms/step - loss: 1.4161 - categorical_accuracy: 0.4062 - val_loss: 1.5705 - val_categorical_accuracy: 0.3125\n",
      "\n",
      "Epoch 00005: saving model to model_init_2020-12-2715_29_07.909081/model-00005-1.41607-0.40616-1.57054-0.31250.h5\n",
      "Epoch 6/30\n",
      "21/21 [==============================] - 12s 591ms/step - loss: 1.2982 - categorical_accuracy: 0.4622 - val_loss: 1.5349 - val_categorical_accuracy: 0.3125\n",
      "\n",
      "Epoch 00006: saving model to model_init_2020-12-2715_29_07.909081/model-00006-1.29823-0.46218-1.53488-0.31250.h5\n",
      "\n",
      "Epoch 00006: ReduceLROnPlateau reducing learning rate to 0.0002500000118743628.\n",
      "Epoch 7/30\n",
      "21/21 [==============================] - 12s 553ms/step - loss: 1.3258 - categorical_accuracy: 0.4230 - val_loss: 1.6234 - val_categorical_accuracy: 0.3125\n",
      "\n",
      "Epoch 00007: saving model to model_init_2020-12-2715_29_07.909081/model-00007-1.32585-0.42297-1.62343-0.31250.h5\n",
      "Epoch 8/30\n",
      "21/21 [==============================] - 12s 588ms/step - loss: 1.2882 - categorical_accuracy: 0.4790 - val_loss: 1.5444 - val_categorical_accuracy: 0.3750\n",
      "\n",
      "Epoch 00008: saving model to model_init_2020-12-2715_29_07.909081/model-00008-1.28818-0.47899-1.54442-0.37500.h5\n",
      "\n",
      "Epoch 00008: ReduceLROnPlateau reducing learning rate to 0.0001250000059371814.\n",
      "Epoch 9/30\n",
      "21/21 [==============================] - 12s 578ms/step - loss: 1.3917 - categorical_accuracy: 0.4062 - val_loss: 1.3203 - val_categorical_accuracy: 0.3750\n",
      "\n",
      "Epoch 00009: saving model to model_init_2020-12-2715_29_07.909081/model-00009-1.39165-0.40616-1.32029-0.37500.h5\n",
      "Epoch 10/30\n",
      "21/21 [==============================] - 11s 543ms/step - loss: 1.1979 - categorical_accuracy: 0.4902 - val_loss: 1.4292 - val_categorical_accuracy: 0.4375\n",
      "\n",
      "Epoch 00010: saving model to model_init_2020-12-2715_29_07.909081/model-00010-1.19787-0.49020-1.42919-0.43750.h5\n",
      "Epoch 11/30\n",
      "21/21 [==============================] - 12s 576ms/step - loss: 1.2864 - categorical_accuracy: 0.4706 - val_loss: 1.4583 - val_categorical_accuracy: 0.3125\n",
      "\n",
      "Epoch 00011: saving model to model_init_2020-12-2715_29_07.909081/model-00011-1.28645-0.47059-1.45830-0.31250.h5\n",
      "\n",
      "Epoch 00011: ReduceLROnPlateau reducing learning rate to 6.25000029685907e-05.\n",
      "Epoch 12/30\n",
      "21/21 [==============================] - 12s 591ms/step - loss: 1.2044 - categorical_accuracy: 0.4986 - val_loss: 1.9371 - val_categorical_accuracy: 0.1875\n",
      "\n",
      "Epoch 00012: saving model to model_init_2020-12-2715_29_07.909081/model-00012-1.20437-0.49860-1.93707-0.18750.h5\n",
      "Epoch 13/30\n",
      "21/21 [==============================] - 12s 571ms/step - loss: 1.2292 - categorical_accuracy: 0.4650 - val_loss: 1.5523 - val_categorical_accuracy: 0.3750\n",
      "\n",
      "Epoch 00013: saving model to model_init_2020-12-2715_29_07.909081/model-00013-1.22918-0.46499-1.55230-0.37500.h5\n",
      "\n",
      "Epoch 00013: ReduceLROnPlateau reducing learning rate to 3.125000148429535e-05.\n",
      "Epoch 14/30\n",
      "21/21 [==============================] - 12s 560ms/step - loss: 1.1765 - categorical_accuracy: 0.5126 - val_loss: 1.5238 - val_categorical_accuracy: 0.1250\n",
      "\n",
      "Epoch 00014: saving model to model_init_2020-12-2715_29_07.909081/model-00014-1.17649-0.51261-1.52384-0.12500.h5\n",
      "Epoch 15/30\n",
      "21/21 [==============================] - 12s 566ms/step - loss: 1.2244 - categorical_accuracy: 0.5378 - val_loss: 1.4116 - val_categorical_accuracy: 0.4375\n",
      "\n",
      "Epoch 00015: saving model to model_init_2020-12-2715_29_07.909081/model-00015-1.22441-0.53782-1.41156-0.43750.h5\n",
      "\n",
      "Epoch 00015: ReduceLROnPlateau reducing learning rate to 1.5625000742147677e-05.\n",
      "Epoch 16/30\n",
      "21/21 [==============================] - 12s 580ms/step - loss: 1.1314 - categorical_accuracy: 0.5490 - val_loss: 1.5416 - val_categorical_accuracy: 0.4375\n",
      "\n",
      "Epoch 00016: saving model to model_init_2020-12-2715_29_07.909081/model-00016-1.13139-0.54902-1.54160-0.43750.h5\n",
      "Epoch 17/30\n",
      "21/21 [==============================] - 12s 566ms/step - loss: 1.2616 - categorical_accuracy: 0.4874 - val_loss: 1.7218 - val_categorical_accuracy: 0.2500\n",
      "\n",
      "Epoch 00017: saving model to model_init_2020-12-2715_29_07.909081/model-00017-1.26162-0.48739-1.72182-0.25000.h5\n",
      "\n",
      "Epoch 00017: ReduceLROnPlateau reducing learning rate to 7.812500371073838e-06.\n",
      "Epoch 18/30\n",
      "21/21 [==============================] - 12s 571ms/step - loss: 1.2591 - categorical_accuracy: 0.4902 - val_loss: 1.8704 - val_categorical_accuracy: 0.1875\n",
      "\n",
      "Epoch 00018: saving model to model_init_2020-12-2715_29_07.909081/model-00018-1.25908-0.49020-1.87035-0.18750.h5\n",
      "Epoch 19/30\n",
      "21/21 [==============================] - 12s 552ms/step - loss: 1.2225 - categorical_accuracy: 0.5154 - val_loss: 1.6392 - val_categorical_accuracy: 0.4375\n",
      "\n",
      "Epoch 00019: saving model to model_init_2020-12-2715_29_07.909081/model-00019-1.22246-0.51541-1.63916-0.43750.h5\n",
      "\n",
      "Epoch 00019: ReduceLROnPlateau reducing learning rate to 3.906250185536919e-06.\n",
      "Epoch 20/30\n",
      "21/21 [==============================] - 12s 562ms/step - loss: 1.2567 - categorical_accuracy: 0.4678 - val_loss: 1.8172 - val_categorical_accuracy: 0.3125\n",
      "\n",
      "Epoch 00020: saving model to model_init_2020-12-2715_29_07.909081/model-00020-1.25669-0.46779-1.81721-0.31250.h5\n",
      "Epoch 21/30\n",
      "21/21 [==============================] - 12s 567ms/step - loss: 1.1837 - categorical_accuracy: 0.5210 - val_loss: 1.7495 - val_categorical_accuracy: 0.2500\n",
      "\n",
      "Epoch 00021: saving model to model_init_2020-12-2715_29_07.909081/model-00021-1.18367-0.52101-1.74946-0.25000.h5\n",
      "\n",
      "Epoch 00021: ReduceLROnPlateau reducing learning rate to 1.9531250927684596e-06.\n",
      "Epoch 22/30\n",
      "21/21 [==============================] - 12s 575ms/step - loss: 1.1564 - categorical_accuracy: 0.5126 - val_loss: 1.2895 - val_categorical_accuracy: 0.5625\n",
      "\n",
      "Epoch 00022: saving model to model_init_2020-12-2715_29_07.909081/model-00022-1.15639-0.51261-1.28950-0.56250.h5\n",
      "Epoch 23/30\n",
      "21/21 [==============================] - 12s 549ms/step - loss: 1.1617 - categorical_accuracy: 0.5322 - val_loss: 1.4012 - val_categorical_accuracy: 0.2500\n",
      "\n",
      "Epoch 00023: saving model to model_init_2020-12-2715_29_07.909081/model-00023-1.16167-0.53221-1.40117-0.25000.h5\n",
      "Epoch 24/30\n",
      "21/21 [==============================] - 13s 599ms/step - loss: 1.2836 - categorical_accuracy: 0.4566 - val_loss: 1.8013 - val_categorical_accuracy: 0.1250\n",
      "\n",
      "Epoch 00024: saving model to model_init_2020-12-2715_29_07.909081/model-00024-1.28364-0.45658-1.80134-0.12500.h5\n",
      "\n",
      "Epoch 00024: ReduceLROnPlateau reducing learning rate to 9.765625463842298e-07.\n",
      "Epoch 25/30\n",
      "21/21 [==============================] - 12s 557ms/step - loss: 1.1449 - categorical_accuracy: 0.5350 - val_loss: 1.4232 - val_categorical_accuracy: 0.4375\n",
      "\n",
      "Epoch 00025: saving model to model_init_2020-12-2715_29_07.909081/model-00025-1.14490-0.53501-1.42324-0.43750.h5\n",
      "Epoch 26/30\n",
      "21/21 [==============================] - 12s 576ms/step - loss: 1.2307 - categorical_accuracy: 0.4650 - val_loss: 1.6120 - val_categorical_accuracy: 0.2500\n",
      "\n",
      "Epoch 00026: saving model to model_init_2020-12-2715_29_07.909081/model-00026-1.23073-0.46499-1.61198-0.25000.h5\n",
      "\n",
      "Epoch 00026: ReduceLROnPlateau reducing learning rate to 4.882812731921149e-07.\n",
      "Epoch 27/30\n",
      "21/21 [==============================] - 12s 565ms/step - loss: 1.2365 - categorical_accuracy: 0.4762 - val_loss: 1.4243 - val_categorical_accuracy: 0.4375\n",
      "\n",
      "Epoch 00027: saving model to model_init_2020-12-2715_29_07.909081/model-00027-1.23651-0.47619-1.42428-0.43750.h5\n",
      "Epoch 28/30\n",
      "21/21 [==============================] - 13s 598ms/step - loss: 1.2089 - categorical_accuracy: 0.5294 - val_loss: 1.4322 - val_categorical_accuracy: 0.3750\n",
      "\n",
      "Epoch 00028: saving model to model_init_2020-12-2715_29_07.909081/model-00028-1.20887-0.52941-1.43222-0.37500.h5\n",
      "\n",
      "Epoch 00028: ReduceLROnPlateau reducing learning rate to 2.4414063659605745e-07.\n",
      "Epoch 29/30\n",
      "21/21 [==============================] - 11s 539ms/step - loss: 1.1341 - categorical_accuracy: 0.5434 - val_loss: 1.4452 - val_categorical_accuracy: 0.4375\n",
      "\n",
      "Epoch 00029: saving model to model_init_2020-12-2715_29_07.909081/model-00029-1.13413-0.54342-1.44523-0.43750.h5\n",
      "Epoch 30/30\n",
      "21/21 [==============================] - 12s 587ms/step - loss: 1.1945 - categorical_accuracy: 0.5350 - val_loss: 2.0783 - val_categorical_accuracy: 0.1875\n",
      "\n",
      "Epoch 00030: saving model to model_init_2020-12-2715_29_07.909081/model-00030-1.19453-0.53501-2.07828-0.18750.h5\n",
      "\n",
      "Epoch 00030: ReduceLROnPlateau reducing learning rate to 1.2207031829802872e-07.\n"
     ]
    }
   ],
   "source": [
    "history13= model13.fit_generator(train_generator, steps_per_epoch=steps_per_epoch, epochs=num_epochs, verbose=1, \n",
    "                    callbacks=callbacks_list, validation_data=val_generator, \n",
    "                    validation_steps=validation_steps, class_weight=None, workers=1, initial_epoch=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Source path =  ./Project_data/val ; batch size = 32\n",
      "Source path = Epoch 1/30\n",
      " ./Project_data/train ; batch size = 32\n",
      "21/21 [==============================] - 100s 5s/step - loss: 1.5989 - categorical_accuracy: 0.3069 - val_loss: 1.3752 - val_categorical_accuracy: 0.4700\n",
      "\n",
      "Epoch 00001: saving model to model_init_2020-12-2715_24_58.874130/model-00001-1.60254-0.30468-1.37515-0.47000.h5\n",
      "Epoch 2/30\n",
      "21/21 [==============================] - 16s 745ms/step - loss: 1.2572 - categorical_accuracy: 0.4948 - val_loss: 0.9024 - val_categorical_accuracy: 0.5625\n",
      "\n",
      "Epoch 00002: saving model to model_init_2020-12-2715_24_58.874130/model-00002-1.25717-0.49482-0.90238-0.56250.h5\n",
      "Epoch 3/30\n",
      "21/21 [==============================] - 17s 826ms/step - loss: 0.9216 - categorical_accuracy: 0.6439 - val_loss: 0.8476 - val_categorical_accuracy: 0.6875\n",
      "\n",
      "Epoch 00003: saving model to model_init_2020-12-2715_24_58.874130/model-00003-0.92622-0.64403-0.84760-0.68750.h5\n",
      "Epoch 4/30\n",
      "21/21 [==============================] - 14s 686ms/step - loss: 0.7633 - categorical_accuracy: 0.7043 - val_loss: 0.5322 - val_categorical_accuracy: 0.8125\n",
      "\n",
      "Epoch 00004: saving model to model_init_2020-12-2715_24_58.874130/model-00004-0.76335-0.70426-0.53223-0.81250.h5\n",
      "Epoch 5/30\n",
      "21/21 [==============================] - 14s 672ms/step - loss: 0.6247 - categorical_accuracy: 0.7619 - val_loss: 0.8470 - val_categorical_accuracy: 0.7500\n",
      "\n",
      "Epoch 00005: saving model to model_init_2020-12-2715_24_58.874130/model-00005-0.62467-0.76190-0.84698-0.75000.h5\n",
      "Epoch 6/30\n",
      "21/21 [==============================] - 14s 662ms/step - loss: 0.6815 - categorical_accuracy: 0.7535 - val_loss: 0.9502 - val_categorical_accuracy: 0.6250\n",
      "\n",
      "Epoch 00006: saving model to model_init_2020-12-2715_24_58.874130/model-00006-0.68149-0.75350-0.95018-0.62500.h5\n",
      "\n",
      "Epoch 00006: ReduceLROnPlateau reducing learning rate to 0.0005000000237487257.\n",
      "Epoch 7/30\n",
      "21/21 [==============================] - 13s 622ms/step - loss: 0.6137 - categorical_accuracy: 0.7619 - val_loss: 0.2668 - val_categorical_accuracy: 0.9375\n",
      "\n",
      "Epoch 00007: saving model to model_init_2020-12-2715_24_58.874130/model-00007-0.61374-0.76190-0.26678-0.93750.h5\n",
      "Epoch 8/30\n",
      "21/21 [==============================] - 14s 651ms/step - loss: 0.3739 - categorical_accuracy: 0.8796 - val_loss: 0.4827 - val_categorical_accuracy: 0.8125\n",
      "\n",
      "Epoch 00008: saving model to model_init_2020-12-2715_24_58.874130/model-00008-0.37395-0.87955-0.48275-0.81250.h5\n",
      "Epoch 9/30\n",
      "21/21 [==============================] - 14s 652ms/step - loss: 0.3096 - categorical_accuracy: 0.8936 - val_loss: 0.4040 - val_categorical_accuracy: 0.8750\n",
      "\n",
      "Epoch 00009: saving model to model_init_2020-12-2715_24_58.874130/model-00009-0.30964-0.89356-0.40398-0.87500.h5\n",
      "\n",
      "Epoch 00009: ReduceLROnPlateau reducing learning rate to 0.0002500000118743628.\n",
      "Epoch 10/30\n",
      "21/21 [==============================] - 14s 684ms/step - loss: 0.2942 - categorical_accuracy: 0.9020 - val_loss: 0.3437 - val_categorical_accuracy: 0.8125\n",
      "\n",
      "Epoch 00010: saving model to model_init_2020-12-2715_24_58.874130/model-00010-0.29419-0.90196-0.34369-0.81250.h5\n",
      "Epoch 11/30\n",
      "21/21 [==============================] - 13s 623ms/step - loss: 0.2420 - categorical_accuracy: 0.9160 - val_loss: 0.4279 - val_categorical_accuracy: 0.8125\n",
      "\n",
      "Epoch 00011: saving model to model_init_2020-12-2715_24_58.874130/model-00011-0.24196-0.91597-0.42786-0.81250.h5\n",
      "\n",
      "Epoch 00011: ReduceLROnPlateau reducing learning rate to 0.0001250000059371814.\n",
      "Epoch 12/30\n",
      "21/21 [==============================] - 14s 648ms/step - loss: 0.2706 - categorical_accuracy: 0.9328 - val_loss: 0.0198 - val_categorical_accuracy: 1.0000\n",
      "\n",
      "Epoch 00012: saving model to model_init_2020-12-2715_24_58.874130/model-00012-0.27061-0.93277-0.01977-1.00000.h5\n",
      "Epoch 13/30\n",
      "21/21 [==============================] - 15s 721ms/step - loss: 0.1941 - categorical_accuracy: 0.9440 - val_loss: 0.2012 - val_categorical_accuracy: 0.8750\n",
      "\n",
      "Epoch 00013: saving model to model_init_2020-12-2715_24_58.874130/model-00013-0.19412-0.94398-0.20117-0.87500.h5\n",
      "Epoch 14/30\n",
      "21/21 [==============================] - 13s 601ms/step - loss: 0.1556 - categorical_accuracy: 0.9580 - val_loss: 0.0545 - val_categorical_accuracy: 1.0000\n",
      "\n",
      "Epoch 00014: saving model to model_init_2020-12-2715_24_58.874130/model-00014-0.15559-0.95798-0.05451-1.00000.h5\n",
      "\n",
      "Epoch 00014: ReduceLROnPlateau reducing learning rate to 6.25000029685907e-05.\n",
      "Epoch 15/30\n",
      "21/21 [==============================] - 15s 695ms/step - loss: 0.1741 - categorical_accuracy: 0.9356 - val_loss: 0.2101 - val_categorical_accuracy: 0.9375\n",
      "\n",
      "Epoch 00015: saving model to model_init_2020-12-2715_24_58.874130/model-00015-0.17411-0.93557-0.21005-0.93750.h5\n",
      "Epoch 16/30\n",
      "21/21 [==============================] - 14s 676ms/step - loss: 0.1655 - categorical_accuracy: 0.9440 - val_loss: 0.3143 - val_categorical_accuracy: 0.9375\n",
      "\n",
      "Epoch 00016: saving model to model_init_2020-12-2715_24_58.874130/model-00016-0.16553-0.94398-0.31427-0.93750.h5\n",
      "\n",
      "Epoch 00016: ReduceLROnPlateau reducing learning rate to 3.125000148429535e-05.\n",
      "Epoch 17/30\n",
      "21/21 [==============================] - 14s 681ms/step - loss: 0.1401 - categorical_accuracy: 0.9692 - val_loss: 0.2919 - val_categorical_accuracy: 0.8750\n",
      "\n",
      "Epoch 00017: saving model to model_init_2020-12-2715_24_58.874130/model-00017-0.14011-0.96919-0.29194-0.87500.h5\n",
      "Epoch 18/30\n",
      "21/21 [==============================] - 14s 677ms/step - loss: 0.1700 - categorical_accuracy: 0.9580 - val_loss: 0.1555 - val_categorical_accuracy: 0.9375\n",
      "\n",
      "Epoch 00018: saving model to model_init_2020-12-2715_24_58.874130/model-00018-0.17001-0.95798-0.15550-0.93750.h5\n",
      "\n",
      "Epoch 00018: ReduceLROnPlateau reducing learning rate to 1.5625000742147677e-05.\n",
      "Epoch 19/30\n",
      "21/21 [==============================] - 13s 640ms/step - loss: 0.0921 - categorical_accuracy: 0.9776 - val_loss: 0.3270 - val_categorical_accuracy: 0.8750\n",
      "\n",
      "Epoch 00019: saving model to model_init_2020-12-2715_24_58.874130/model-00019-0.09214-0.97759-0.32698-0.87500.h5\n",
      "Epoch 20/30\n",
      "21/21 [==============================] - 14s 653ms/step - loss: 0.1236 - categorical_accuracy: 0.9636 - val_loss: 0.0780 - val_categorical_accuracy: 0.9375\n",
      "\n",
      "Epoch 00020: saving model to model_init_2020-12-2715_24_58.874130/model-00020-0.12362-0.96359-0.07796-0.93750.h5\n",
      "\n",
      "Epoch 00020: ReduceLROnPlateau reducing learning rate to 7.812500371073838e-06.\n",
      "Epoch 21/30\n",
      "21/21 [==============================] - 14s 669ms/step - loss: 0.1300 - categorical_accuracy: 0.9608 - val_loss: 0.3224 - val_categorical_accuracy: 0.8750\n",
      "\n",
      "Epoch 00021: saving model to model_init_2020-12-2715_24_58.874130/model-00021-0.13002-0.96078-0.32241-0.87500.h5\n",
      "Epoch 22/30\n",
      "21/21 [==============================] - 14s 681ms/step - loss: 0.1198 - categorical_accuracy: 0.9720 - val_loss: 0.2812 - val_categorical_accuracy: 0.9375\n",
      "\n",
      "Epoch 00022: saving model to model_init_2020-12-2715_24_58.874130/model-00022-0.11980-0.97199-0.28124-0.93750.h5\n",
      "\n",
      "Epoch 00022: ReduceLROnPlateau reducing learning rate to 3.906250185536919e-06.\n",
      "Epoch 23/30\n",
      "21/21 [==============================] - 19s 888ms/step - loss: 0.1338 - categorical_accuracy: 0.9636 - val_loss: 0.4111 - val_categorical_accuracy: 0.8125\n",
      "\n",
      "Epoch 00023: saving model to model_init_2020-12-2715_24_58.874130/model-00023-0.13378-0.96359-0.41109-0.81250.h5\n",
      "Epoch 24/30\n",
      "21/21 [==============================] - 17s 831ms/step - loss: 0.1158 - categorical_accuracy: 0.9636 - val_loss: 0.0966 - val_categorical_accuracy: 0.9375\n",
      "\n",
      "Epoch 00024: saving model to model_init_2020-12-2715_24_58.874130/model-00024-0.11584-0.96359-0.09659-0.93750.h5\n",
      "\n",
      "Epoch 00024: ReduceLROnPlateau reducing learning rate to 1.9531250927684596e-06.\n",
      "Epoch 25/30\n",
      "21/21 [==============================] - 15s 728ms/step - loss: 0.1531 - categorical_accuracy: 0.9496 - val_loss: 0.1552 - val_categorical_accuracy: 0.8750\n",
      "\n",
      "Epoch 00025: saving model to model_init_2020-12-2715_24_58.874130/model-00025-0.15306-0.94958-0.15517-0.87500.h5\n",
      "Epoch 26/30\n",
      "21/21 [==============================] - 13s 634ms/step - loss: 0.1206 - categorical_accuracy: 0.9664 - val_loss: 0.1240 - val_categorical_accuracy: 0.9375\n",
      "\n",
      "Epoch 00026: saving model to model_init_2020-12-2715_24_58.874130/model-00026-0.12059-0.96639-0.12404-0.93750.h5\n",
      "\n",
      "Epoch 00026: ReduceLROnPlateau reducing learning rate to 9.765625463842298e-07.\n",
      "Epoch 27/30\n",
      "21/21 [==============================] - 11s 546ms/step - loss: 0.1528 - categorical_accuracy: 0.9524 - val_loss: 0.1507 - val_categorical_accuracy: 0.9375\n",
      "\n",
      "Epoch 00027: saving model to model_init_2020-12-2715_24_58.874130/model-00027-0.15280-0.95238-0.15065-0.93750.h5\n",
      "Epoch 28/30\n",
      "21/21 [==============================] - 12s 591ms/step - loss: 0.1452 - categorical_accuracy: 0.9608 - val_loss: 0.4048 - val_categorical_accuracy: 0.8750\n",
      "\n",
      "Epoch 00028: saving model to model_init_2020-12-2715_24_58.874130/model-00028-0.14524-0.96078-0.40480-0.87500.h5\n",
      "\n",
      "Epoch 00028: ReduceLROnPlateau reducing learning rate to 4.882812731921149e-07.\n",
      "Epoch 29/30\n",
      "21/21 [==============================] - 13s 597ms/step - loss: 0.1477 - categorical_accuracy: 0.9496 - val_loss: 0.1362 - val_categorical_accuracy: 0.8750\n",
      "\n",
      "Epoch 00029: saving model to model_init_2020-12-2715_24_58.874130/model-00029-0.14768-0.94958-0.13625-0.87500.h5\n",
      "Epoch 30/30\n",
      "21/21 [==============================] - 11s 532ms/step - loss: 0.1306 - categorical_accuracy: 0.9636 - val_loss: 0.2522 - val_categorical_accuracy: 0.9375\n",
      "\n",
      "Epoch 00030: saving model to model_init_2020-12-2715_24_58.874130/model-00030-0.13061-0.96359-0.25222-0.93750.h5\n",
      "\n",
      "Epoch 00030: ReduceLROnPlateau reducing learning rate to 2.4414063659605745e-07.\n"
     ]
    }
   ],
   "source": [
    "history14= model14.fit_generator(train_generator, steps_per_epoch=steps_per_epoch, epochs=num_epochs, verbose=1, \n",
    "                    callbacks=callbacks_list, validation_data=val_generator, \n",
    "                    validation_steps=validation_steps, class_weight=None, workers=1, initial_epoch=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Source path =  ./Project_data/val ; batch size = 32\n",
      "Source path =  ./Project_data/train ; batch size = 32\n",
      "Epoch 1/30\n",
      "21/21 [==============================] - 30s 1s/step - loss: 1.7309 - categorical_accuracy: 0.2870 - val_loss: 1.3849 - val_categorical_accuracy: 0.4100\n",
      "\n",
      "Epoch 00001: saving model to model_init_2020-12-2715_50_52.109010/model-00001-1.73513-0.28507-1.38494-0.41000.h5\n",
      "Epoch 2/30\n",
      "21/21 [==============================] - 12s 593ms/step - loss: 1.1822 - categorical_accuracy: 0.5259 - val_loss: 1.0706 - val_categorical_accuracy: 0.5625\n",
      "\n",
      "Epoch 00002: saving model to model_init_2020-12-2715_50_52.109010/model-00002-1.18225-0.52588-1.07058-0.56250.h5\n",
      "Epoch 3/30\n",
      "21/21 [==============================] - 12s 556ms/step - loss: 0.8409 - categorical_accuracy: 0.6697 - val_loss: 1.0148 - val_categorical_accuracy: 0.5000\n",
      "\n",
      "Epoch 00003: saving model to model_init_2020-12-2715_50_52.109010/model-00003-0.85009-0.66745-1.01482-0.50000.h5\n",
      "Epoch 4/30\n",
      "21/21 [==============================] - 13s 634ms/step - loss: 0.7593 - categorical_accuracy: 0.7143 - val_loss: 0.7427 - val_categorical_accuracy: 0.7500\n",
      "\n",
      "Epoch 00004: saving model to model_init_2020-12-2715_50_52.109010/model-00004-0.75929-0.71429-0.74268-0.75000.h5\n",
      "Epoch 5/30\n",
      "21/21 [==============================] - 12s 587ms/step - loss: 0.5524 - categorical_accuracy: 0.8179 - val_loss: 0.9348 - val_categorical_accuracy: 0.4375\n",
      "\n",
      "Epoch 00005: saving model to model_init_2020-12-2715_50_52.109010/model-00005-0.55241-0.81793-0.93475-0.43750.h5\n",
      "Epoch 6/30\n",
      "21/21 [==============================] - 12s 562ms/step - loss: 0.6514 - categorical_accuracy: 0.7703 - val_loss: 1.1878 - val_categorical_accuracy: 0.6875\n",
      "\n",
      "Epoch 00006: saving model to model_init_2020-12-2715_50_52.109010/model-00006-0.65140-0.77031-1.18775-0.68750.h5\n",
      "\n",
      "Epoch 00006: ReduceLROnPlateau reducing learning rate to 0.0005000000237487257.\n",
      "Epoch 7/30\n",
      "21/21 [==============================] - 10s 496ms/step - loss: 0.4181 - categorical_accuracy: 0.8655 - val_loss: 1.3274 - val_categorical_accuracy: 0.5625\n",
      "\n",
      "Epoch 00007: saving model to model_init_2020-12-2715_50_52.109010/model-00007-0.41810-0.86555-1.32741-0.56250.h5\n",
      "Epoch 8/30\n",
      "21/21 [==============================] - 12s 563ms/step - loss: 0.3661 - categorical_accuracy: 0.8627 - val_loss: 0.9744 - val_categorical_accuracy: 0.7500\n",
      "\n",
      "Epoch 00008: saving model to model_init_2020-12-2715_50_52.109010/model-00008-0.36607-0.86275-0.97438-0.75000.h5\n",
      "\n",
      "Epoch 00008: ReduceLROnPlateau reducing learning rate to 0.0002500000118743628.\n",
      "Epoch 9/30\n",
      "21/21 [==============================] - 12s 578ms/step - loss: 0.2506 - categorical_accuracy: 0.9300 - val_loss: 0.5187 - val_categorical_accuracy: 0.8125\n",
      "\n",
      "Epoch 00009: saving model to model_init_2020-12-2715_50_52.109010/model-00009-0.25064-0.92997-0.51869-0.81250.h5\n",
      "Epoch 10/30\n",
      "21/21 [==============================] - 12s 579ms/step - loss: 0.2141 - categorical_accuracy: 0.9356 - val_loss: 0.3087 - val_categorical_accuracy: 0.8750\n",
      "\n",
      "Epoch 00010: saving model to model_init_2020-12-2715_50_52.109010/model-00010-0.21405-0.93557-0.30868-0.87500.h5\n",
      "Epoch 11/30\n",
      "21/21 [==============================] - 12s 570ms/step - loss: 0.1791 - categorical_accuracy: 0.9524 - val_loss: 0.2784 - val_categorical_accuracy: 0.8750\n",
      "\n",
      "Epoch 00011: saving model to model_init_2020-12-2715_50_52.109010/model-00011-0.17912-0.95238-0.27844-0.87500.h5\n",
      "Epoch 12/30\n",
      "21/21 [==============================] - 12s 568ms/step - loss: 0.1872 - categorical_accuracy: 0.9524 - val_loss: 0.2608 - val_categorical_accuracy: 0.8750\n",
      "\n",
      "Epoch 00012: saving model to model_init_2020-12-2715_50_52.109010/model-00012-0.18720-0.95238-0.26083-0.87500.h5\n",
      "Epoch 13/30\n",
      "21/21 [==============================] - 12s 576ms/step - loss: 0.1536 - categorical_accuracy: 0.9636 - val_loss: 0.9882 - val_categorical_accuracy: 0.6875\n",
      "\n",
      "Epoch 00013: saving model to model_init_2020-12-2715_50_52.109010/model-00013-0.15364-0.96359-0.98816-0.68750.h5\n",
      "Epoch 14/30\n",
      "21/21 [==============================] - 11s 511ms/step - loss: 0.1309 - categorical_accuracy: 0.9608 - val_loss: 0.5097 - val_categorical_accuracy: 0.8750\n",
      "\n",
      "Epoch 00014: saving model to model_init_2020-12-2715_50_52.109010/model-00014-0.13088-0.96078-0.50967-0.87500.h5\n",
      "\n",
      "Epoch 00014: ReduceLROnPlateau reducing learning rate to 0.0001250000059371814.\n",
      "Epoch 15/30\n",
      "21/21 [==============================] - 12s 588ms/step - loss: 0.1103 - categorical_accuracy: 0.9664 - val_loss: 0.8560 - val_categorical_accuracy: 0.8125\n",
      "\n",
      "Epoch 00015: saving model to model_init_2020-12-2715_50_52.109010/model-00015-0.11034-0.96639-0.85598-0.81250.h5\n",
      "Epoch 16/30\n",
      "21/21 [==============================] - 12s 556ms/step - loss: 0.1184 - categorical_accuracy: 0.9720 - val_loss: 0.7113 - val_categorical_accuracy: 0.8125\n",
      "\n",
      "Epoch 00016: saving model to model_init_2020-12-2715_50_52.109010/model-00016-0.11841-0.97199-0.71125-0.81250.h5\n",
      "\n",
      "Epoch 00016: ReduceLROnPlateau reducing learning rate to 6.25000029685907e-05.\n",
      "Epoch 17/30\n",
      "21/21 [==============================] - 12s 563ms/step - loss: 0.0991 - categorical_accuracy: 0.9720 - val_loss: 0.2690 - val_categorical_accuracy: 0.8750\n",
      "\n",
      "Epoch 00017: saving model to model_init_2020-12-2715_50_52.109010/model-00017-0.09910-0.97199-0.26897-0.87500.h5\n",
      "Epoch 18/30\n",
      "21/21 [==============================] - 12s 570ms/step - loss: 0.0949 - categorical_accuracy: 0.9720 - val_loss: 0.6472 - val_categorical_accuracy: 0.6875\n",
      "\n",
      "Epoch 00018: saving model to model_init_2020-12-2715_50_52.109010/model-00018-0.09493-0.97199-0.64723-0.68750.h5\n",
      "\n",
      "Epoch 00018: ReduceLROnPlateau reducing learning rate to 3.125000148429535e-05.\n",
      "Epoch 19/30\n",
      "21/21 [==============================] - 12s 561ms/step - loss: 0.0704 - categorical_accuracy: 0.9832 - val_loss: 0.3636 - val_categorical_accuracy: 0.8750\n",
      "\n",
      "Epoch 00019: saving model to model_init_2020-12-2715_50_52.109010/model-00019-0.07035-0.98319-0.36359-0.87500.h5\n",
      "Epoch 20/30\n",
      "21/21 [==============================] - 12s 564ms/step - loss: 0.0911 - categorical_accuracy: 0.9804 - val_loss: 0.0483 - val_categorical_accuracy: 1.0000\n",
      "\n",
      "Epoch 00020: saving model to model_init_2020-12-2715_50_52.109010/model-00020-0.09108-0.98039-0.04831-1.00000.h5\n",
      "Epoch 21/30\n",
      "21/21 [==============================] - 12s 560ms/step - loss: 0.0767 - categorical_accuracy: 0.9804 - val_loss: 0.8440 - val_categorical_accuracy: 0.7500\n",
      "\n",
      "Epoch 00021: saving model to model_init_2020-12-2715_50_52.109010/model-00021-0.07671-0.98039-0.84397-0.75000.h5\n",
      "Epoch 22/30\n",
      "21/21 [==============================] - 12s 554ms/step - loss: 0.1342 - categorical_accuracy: 0.9580 - val_loss: 0.7616 - val_categorical_accuracy: 0.7500\n",
      "\n",
      "Epoch 00022: saving model to model_init_2020-12-2715_50_52.109010/model-00022-0.13420-0.95798-0.76160-0.75000.h5\n",
      "\n",
      "Epoch 00022: ReduceLROnPlateau reducing learning rate to 1.5625000742147677e-05.\n",
      "Epoch 23/30\n",
      "21/21 [==============================] - 12s 551ms/step - loss: 0.1052 - categorical_accuracy: 0.9748 - val_loss: 0.4815 - val_categorical_accuracy: 0.8125\n",
      "\n",
      "Epoch 00023: saving model to model_init_2020-12-2715_50_52.109010/model-00023-0.10524-0.97479-0.48154-0.81250.h5\n",
      "Epoch 24/30\n",
      "21/21 [==============================] - 12s 565ms/step - loss: 0.0862 - categorical_accuracy: 0.9916 - val_loss: 0.1607 - val_categorical_accuracy: 0.9375\n",
      "\n",
      "Epoch 00024: saving model to model_init_2020-12-2715_50_52.109010/model-00024-0.08618-0.99160-0.16070-0.93750.h5\n",
      "\n",
      "Epoch 00024: ReduceLROnPlateau reducing learning rate to 7.812500371073838e-06.\n",
      "Epoch 25/30\n",
      "21/21 [==============================] - 12s 555ms/step - loss: 0.1049 - categorical_accuracy: 0.9720 - val_loss: 0.4280 - val_categorical_accuracy: 0.8125\n",
      "\n",
      "Epoch 00025: saving model to model_init_2020-12-2715_50_52.109010/model-00025-0.10490-0.97199-0.42800-0.81250.h5\n",
      "Epoch 26/30\n",
      "21/21 [==============================] - 12s 563ms/step - loss: 0.0828 - categorical_accuracy: 0.9748 - val_loss: 0.0300 - val_categorical_accuracy: 1.0000\n",
      "\n",
      "Epoch 00026: saving model to model_init_2020-12-2715_50_52.109010/model-00026-0.08275-0.97479-0.03001-1.00000.h5\n",
      "Epoch 27/30\n",
      "21/21 [==============================] - 11s 534ms/step - loss: 0.0859 - categorical_accuracy: 0.9776 - val_loss: 0.6826 - val_categorical_accuracy: 0.8750\n",
      "\n",
      "Epoch 00027: saving model to model_init_2020-12-2715_50_52.109010/model-00027-0.08594-0.97759-0.68259-0.87500.h5\n",
      "Epoch 28/30\n",
      "21/21 [==============================] - 12s 572ms/step - loss: 0.0544 - categorical_accuracy: 0.9888 - val_loss: 0.3004 - val_categorical_accuracy: 0.8125\n",
      "\n",
      "Epoch 00028: saving model to model_init_2020-12-2715_50_52.109010/model-00028-0.05436-0.98880-0.30044-0.81250.h5\n",
      "\n",
      "Epoch 00028: ReduceLROnPlateau reducing learning rate to 3.906250185536919e-06.\n",
      "Epoch 29/30\n",
      "21/21 [==============================] - 12s 585ms/step - loss: 0.0827 - categorical_accuracy: 0.9916 - val_loss: 0.3685 - val_categorical_accuracy: 0.9375\n",
      "\n",
      "Epoch 00029: saving model to model_init_2020-12-2715_50_52.109010/model-00029-0.08274-0.99160-0.36853-0.93750.h5\n",
      "Epoch 30/30\n",
      "21/21 [==============================] - 11s 533ms/step - loss: 0.0793 - categorical_accuracy: 0.9832 - val_loss: 0.1973 - val_categorical_accuracy: 0.9375\n",
      "\n",
      "Epoch 00030: saving model to model_init_2020-12-2715_50_52.109010/model-00030-0.07929-0.98319-0.19732-0.93750.h5\n",
      "\n",
      "Epoch 00030: ReduceLROnPlateau reducing learning rate to 1.9531250927684596e-06.\n"
     ]
    }
   ],
   "source": [
    "history15= model15.fit_generator(train_generator, steps_per_epoch=steps_per_epoch, epochs=num_epochs, verbose=1, \n",
    "                    callbacks=callbacks_list, validation_data=val_generator, \n",
    "                    validation_steps=validation_steps, class_weight=None, workers=1, initial_epoch=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}