STFT_CNN_benchmark.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "STFT_CNN_benchmark.ipynb",
      "version": "0.3.2",
      "provenance": [],
      "collapsed_sections": [],
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/zabir-nabil/eeg-rsenet/blob/master/STFT_CNN_benchmark.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "metadata": {
        "id": "RHjCcXcUw2lf",
        "colab_type": "code",
        "outputId": "cd488850-2e7d-462f-a300-9860632c0fb5",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        }
      },
      "cell_type": "code",
      "source": [
        "print('zabiralnazi@yahoo.com')"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "zabiralnazi@yahoo.com\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "id": "rFG1t61VxBej",
        "colab_type": "code",
        "outputId": "26d3276a-5d59-4d3c-b00a-4456c2cde524",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 55
        }
      },
      "cell_type": "code",
      "source": [
        "# Load the Drive helper and mount\n",
        "from google.colab import drive\n",
        "\n",
        "# This will prompt for authorization.\n",
        "drive.mount('/content/drive')"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Drive already mounted at /content/drive; to attempt to forcibly remount, call drive.mount(\"/content/drive\", force_remount=True).\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "id": "nnBYnlwk1YYt",
        "colab_type": "code",
        "outputId": "96b2144d-d5f4-4c93-e90e-8435759d9da2",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        }
      },
      "cell_type": "code",
      "source": [
        "% cd /content/drive/My Drive/lvl 2 image/"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "/content/drive/My Drive/lvl 2 image\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "id": "BUFl0_D54JCC",
        "colab_type": "code",
        "outputId": "39998629-d4d2-4626-c897-c90d722b70a2",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        }
      },
      "cell_type": "code",
      "source": [
        "'/content/drive/My Drive/lvl 1 image/mi_107_2_[1].png'[-9]"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "'2'"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 4
        }
      ]
    },
    {
      "metadata": {
        "id": "KjzVBE742IN7",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "import glob\n",
        "\n",
        "cls1_ch1 = []\n",
        "cls1_ch2 = []\n",
        "cls1_ch3 = []\n",
        "\n",
        "cls2_ch1 = []\n",
        "cls2_ch2 = []\n",
        "cls2_ch3 = []\n",
        "\n",
        "for f in glob.glob('/content/drive/My Drive/lvl 1 image/*.png'):\n",
        "  if(f[-9]=='0'):\n",
        "    cls1_ch1.append(f)\n",
        "  if(f[-9]=='1'):\n",
        "    cls1_ch2.append(f)\n",
        "  if(f[-9]=='2'):\n",
        "    cls1_ch3.append(f)\n",
        "    \n",
        "for f in glob.glob('/content/drive/My Drive/lvl 2 image/*.png'):\n",
        "  if(f[-9]=='0'):\n",
        "    cls2_ch1.append(f)\n",
        "  if(f[-9]=='1'):\n",
        "    cls2_ch2.append(f)\n",
        "  if(f[-9]=='2'):\n",
        "    cls2_ch3.append(f)\n",
        "    "
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "TNz3owUNZCTm",
        "colab_type": "code",
        "outputId": "2dcb7ebb-e268-4fec-f628-78968143c3b6",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        }
      },
      "cell_type": "code",
      "source": [
        "len(cls2_ch1)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "70"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 6
        }
      ]
    },
    {
      "metadata": {
        "id": "kkBP8Go9390J",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "cls1_ch1.sort()\n",
        "cls1_ch2.sort()\n",
        "cls1_ch3.sort()\n",
        "\n",
        "cls2_ch1.sort()\n",
        "cls2_ch2.sort()\n",
        "cls2_ch3.sort()"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "9Ia_Jzy64tWu",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "import cv2\n",
        "\n",
        "x1 = []\n",
        "x2 = []\n",
        "x3 = []\n",
        "\n",
        "y = []\n",
        "\n",
        "for i in range(len(cls1_ch1)):\n",
        "  x1.append(cv2.resize(cv2.imread(cls1_ch1[i]), (224, 224)))\n",
        "  x2.append(cv2.resize(cv2.imread(cls1_ch2[i]), (224, 224)))\n",
        "  x3.append(cv2.resize(cv2.imread(cls1_ch3[i]), (224, 224)))\n",
        "  \n",
        "  y.append([1, 0])\n",
        "  \n",
        "for i in range(len(cls2_ch1)):\n",
        "  x1.append(cv2.resize(cv2.imread(cls2_ch1[i]), (224, 224)))\n",
        "  x2.append(cv2.resize(cv2.imread(cls2_ch2[i]), (224, 224)))\n",
        "  x3.append(cv2.resize(cv2.imread(cls2_ch3[i]), (224, 224)))\n",
        "  \n",
        "  y.append([0, 1])"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "ZQWs3ZvCEuTw",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "# Augmentor can't be used directly due to multi-channel"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "HNPyDetpE7sI",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "##################################------------------------------------------------######################################"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "TkpJgIsw9z6t",
        "colab_type": "code",
        "outputId": "30200043-5ea1-4e47-812a-0902e67edca4",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 146
        }
      },
      "cell_type": "code",
      "source": [
        "# Augmentation part\n",
        "\n",
        "# install augmentation library\n",
        "! pip install Augmentor"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Requirement already satisfied: Augmentor in /usr/local/lib/python3.6/dist-packages (0.2.3)\n",
            "Requirement already satisfied: future>=0.16.0 in /usr/local/lib/python3.6/dist-packages (from Augmentor) (0.16.0)\n",
            "Requirement already satisfied: numpy>=1.11.0 in /usr/local/lib/python3.6/dist-packages (from Augmentor) (1.14.6)\n",
            "Requirement already satisfied: tqdm>=4.9.0 in /usr/local/lib/python3.6/dist-packages (from Augmentor) (4.28.1)\n",
            "Requirement already satisfied: Pillow>=4.0.0 in /usr/local/lib/python3.6/dist-packages (from Augmentor) (4.0.0)\n",
            "Requirement already satisfied: olefile in /usr/local/lib/python3.6/dist-packages (from Pillow>=4.0.0->Augmentor) (0.46)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "id": "kgJjI5vR95vw",
        "colab_type": "code",
        "outputId": "38514fd5-ea6f-4998-d623-adb81d891a00",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 73
        }
      },
      "cell_type": "code",
      "source": [
        "import Augmentor\n",
        "p = Augmentor.Pipeline(\"/content/drive/My Drive/lvl 1 image/\")\n",
        "# Point to a directory containing ground truth data.\n",
        "# Images with the same file names will be added as ground truth data\n",
        "# and augmented in parallel to the original data.\n",
        "\n",
        "# Add operations to the pipeline as normal:\n",
        "p.rotate(probability=0.2, max_left_rotation=5, max_right_rotation=5)\n",
        "p.flip_left_right(probability=0.2)\n",
        "p.zoom_random(probability=0.1, percentage_area=0.8)\n",
        "p.flip_top_bottom(probability=0.3)\n",
        "p.gaussian_distortion(probability=0.05, grid_width=4, grid_height=4, magnitude=3, corner='bell', method='in', mex=0.5, mey=0.5, sdx=0.05, sdy=0.05)\n",
        "p.random_brightness(probability=0.05, min_factor=0.7, max_factor=1.3)\n",
        "p.random_color(probability=0.05, min_factor=0.6, max_factor=0.9)\n",
        "p.random_contrast(probability=0.05, min_factor=0.6, max_factor=0.9)\n",
        "p.random_distortion(probability=0.2, grid_width=4, grid_height=4, magnitude=2)\n",
        "\n",
        "p.sample(1000) # change this number to generate different number of augmented variations"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "\rExecuting Pipeline:   0%|          | 0/1000 [00:00<?, ? Samples/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Initialised with 210 image(s) found.\n",
            "Output directory set to /content/drive/My Drive/lvl 1 image/output."
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "Processing <PIL.Image.Image image mode=RGBA size=800x800 at 0x7FF833A1A4A8>: 100%|██████████| 1000/1000 [01:51<00:00,  9.99 Samples/s]\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "metadata": {
        "id": "emEjQrYYBb2v",
        "colab_type": "code",
        "outputId": "0dff4fe8-ddd4-4dc7-e564-aeecffbcca1d",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 73
        }
      },
      "cell_type": "code",
      "source": [
        "p = Augmentor.Pipeline(\"/content/drive/My Drive/lvl 2 image/\")\n",
        "# Point to a directory containing ground truth data.\n",
        "# Images with the same file names will be added as ground truth data\n",
        "# and augmented in parallel to the original data.\n",
        "\n",
        "# Add operations to the pipeline as normal:\n",
        "p.rotate(probability=0.2, max_left_rotation=5, max_right_rotation=5)\n",
        "p.flip_left_right(probability=0.2)\n",
        "p.zoom_random(probability=0.1, percentage_area=0.8)\n",
        "p.flip_top_bottom(probability=0.3)\n",
        "p.gaussian_distortion(probability=0.05, grid_width=4, grid_height=4, magnitude=3, corner='bell', method='in', mex=0.5, mey=0.5, sdx=0.05, sdy=0.05)\n",
        "p.random_brightness(probability=0.05, min_factor=0.7, max_factor=1.3)\n",
        "p.random_color(probability=0.05, min_factor=0.6, max_factor=0.9)\n",
        "p.random_contrast(probability=0.05, min_factor=0.6, max_factor=0.9)\n",
        "p.random_distortion(probability=0.2, grid_width=4, grid_height=4, magnitude=2)\n",
        "\n",
        "p.sample(1000) # change this number to generate different number of augmented variations"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "\rExecuting Pipeline:   0%|          | 0/1000 [00:00<?, ? Samples/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Initialised with 210 image(s) found.\n",
            "Output directory set to /content/drive/My Drive/lvl 2 image/output."
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "Processing <PIL.PngImagePlugin.PngImageFile image mode=RGBA size=800x800 at 0x7FF83215FC88>: 100%|██████████| 1000/1000 [01:46<00:00,  9.37 Samples/s]\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "metadata": {
        "id": "IXJ80DX9_FOf",
        "colab_type": "code",
        "outputId": "f633c6b0-9183-4b29-9545-18a2bbae0aa7",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        }
      },
      "cell_type": "code",
      "source": [
        "% cd /content/drive/My Drive/lvl 1 image/output/"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "/content/drive/My Drive/lvl 1 image/output\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "id": "bskN0EO-Ckr4",
        "colab_type": "code",
        "outputId": "38745be6-d64f-4276-bba8-0cce58b827ed",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        }
      },
      "cell_type": "code",
      "source": [
        "% cd /content/drive/My Drive/lvl 2 image/output/"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "/content/drive/My Drive/lvl 2 image/output\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "id": "lEhKEd2pAQNX",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "def get_ch(s):\n",
        "  i = 0\n",
        "  for c in s:\n",
        "    if c == '[':\n",
        "      return s[i-2]\n",
        "    i += 1"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "3KWcYgZCAg7e",
        "colab_type": "code",
        "outputId": "d0b1d4ae-2184-42a6-d1ca-6b4e7e124561",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        }
      },
      "cell_type": "code",
      "source": [
        "get_ch('/content/drive/My Drive/lvl 1 image/output/lvl 1 image_original_mi_136_1_[1].png_8a2d130c-56f4-40b0-b5c7-ae8e5704bc3b.png')"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "'1'"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 21
        }
      ]
    },
    {
      "metadata": {
        "id": "YB-pZoPA_L04",
        "colab_type": "code",
        "outputId": "37d93f9a-b61a-481e-d360-95561f9435a6",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        }
      },
      "cell_type": "code",
      "source": [
        "#for f in glob.glob('/content/drive/My Drive/lvl 2 image/output/*.png'):\n",
        "#  print(f)\n",
        "\n",
        "get_ch('/content/drive/My Drive/lvl 1 image/output/lvl 1 image_original_mi_136_3_[1].png_8a2d130c-56f4-40b0-b5c7-ae8e5704bc3b.png')"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "'3'"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 27
        }
      ]
    },
    {
      "metadata": {
        "id": "7tQASQwoBUNf",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "cls1_ch1 = []\n",
        "cls1_ch2 = []\n",
        "cls1_ch3 = []\n",
        "\n",
        "cls2_ch1 = []\n",
        "cls2_ch2 = []\n",
        "cls2_ch3 = []\n",
        "\n",
        "for f in glob.glob('/content/drive/My Drive/lvl 1 image/output/*.png'):\n",
        "  if(get_ch(f)=='0'):\n",
        "    cls1_ch1.append(f)\n",
        "  if(get_ch(f)=='1'):\n",
        "    cls1_ch2.append(f)\n",
        "  if(get_ch(f)=='2'):\n",
        "    cls1_ch3.append(f)\n",
        "    \n",
        "for f in glob.glob('/content/drive/My Drive/lvl 2 image/output/*.png'):\n",
        "  if(get_ch(f)=='0'):\n",
        "    cls2_ch1.append(f)\n",
        "  if(get_ch(f)=='1'):\n",
        "    cls2_ch2.append(f)\n",
        "  if(get_ch(f)=='2'):\n",
        "    cls2_ch3.append(f)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "Ll66g2AWDkYP",
        "colab_type": "code",
        "outputId": "076d2ab9-29da-4036-e3db-81b8bd29609b",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 126
        }
      },
      "cell_type": "code",
      "source": [
        "print(len(cls1_ch1))\n",
        "print(len(cls1_ch2))\n",
        "print(len(cls1_ch3))\n",
        "print(len(cls2_ch1))\n",
        "print(len(cls2_ch2))\n",
        "print(len(cls2_ch3))"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "348\n",
            "327\n",
            "325\n",
            "309\n",
            "356\n",
            "335\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "id": "0LZVJR8gE0xo",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "##################################------------------------------------------------######################################"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "GVpqtP6iMGQ5",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "# customization for multi-channel"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "kPEs4R2WZTkd",
        "colab_type": "code",
        "outputId": "a97e5dad-d7c6-43a4-de76-a1cac75db6af",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 146
        }
      },
      "cell_type": "code",
      "source": [
        "! pip install Augmentor "
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Requirement already satisfied: Augmentor in /usr/local/lib/python3.6/dist-packages (0.2.3)\n",
            "Requirement already satisfied: future>=0.16.0 in /usr/local/lib/python3.6/dist-packages (from Augmentor) (0.16.0)\n",
            "Requirement already satisfied: Pillow>=4.0.0 in /usr/local/lib/python3.6/dist-packages (from Augmentor) (4.0.0)\n",
            "Requirement already satisfied: tqdm>=4.9.0 in /usr/local/lib/python3.6/dist-packages (from Augmentor) (4.28.1)\n",
            "Requirement already satisfied: numpy>=1.11.0 in /usr/local/lib/python3.6/dist-packages (from Augmentor) (1.14.6)\n",
            "Requirement already satisfied: olefile in /usr/local/lib/python3.6/dist-packages (from Pillow>=4.0.0->Augmentor) (0.46)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "id": "5sJ42H30TMqD",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "z = list(zip(x1,x2,x3))"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "mgKxgkF2UDSi",
        "colab_type": "code",
        "outputId": "bed2ab7a-2778-4b47-e868-e596db10b66c",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        }
      },
      "cell_type": "code",
      "source": [
        "print(len((z)))"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "140\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "id": "BmqvitIXXZeE",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "def unzip(z):\n",
        "  x1_p = [a[0] for a in z]\n",
        "  x2_p = [a[1] for a in z]\n",
        "  x3_p = [a[2] for a in z]\n",
        "  return x1_p, x2_p, x3_p"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "NkX-yn3ISo78",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "import random\n",
        "random.seed(1997)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "gY7Pwst5J9_C",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "import Augmentor\n",
        "\n",
        "p = Augmentor.DataPipeline(z, y)\n",
        "\n",
        "p.rotate(probability=0.2, max_left_rotation=5, max_right_rotation=5)\n",
        "p.flip_left_right(probability=0.2)\n",
        "p.zoom_random(probability=0.1, percentage_area=0.8)\n",
        "p.flip_top_bottom(probability=0.3)\n",
        "p.gaussian_distortion(probability=0.05, grid_width=4, grid_height=4, magnitude=3, corner='bell', method='in', mex=0.5, mey=0.5, sdx=0.05, sdy=0.05)\n",
        "p.random_brightness(probability=0.05, min_factor=0.7, max_factor=1.3)\n",
        "p.random_color(probability=0.05, min_factor=0.6, max_factor=0.9)\n",
        "p.random_contrast(probability=0.05, min_factor=0.6, max_factor=0.9)\n",
        "p.random_distortion(probability=0.2, grid_width=4, grid_height=4, magnitude=2)\n",
        "\n",
        "augmented_images, labels = p.sample(1000)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "IRSPeKVrLSTi",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "x1_p, x2_p, x3_p = unzip(augmented_images)\n",
        "y_p = labels"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "wR4L9PNoQWRz",
        "colab_type": "code",
        "outputId": "abecce56-35b8-4724-9970-23ad7c6ca6cb",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        }
      },
      "cell_type": "code",
      "source": [
        "print(augmented_images[9][2].shape)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "(224, 224, 3)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "id": "R7t5XS9cX5hs",
        "colab_type": "code",
        "outputId": "ee3578ee-b24b-484f-87be-ddeeafc109e5",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 72
        }
      },
      "cell_type": "code",
      "source": [
        "print(len(x1_p))\n",
        "print(len(x2_p))\n",
        "print(len(x3_p))"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "1000\n",
            "1000\n",
            "1000\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "id": "CM1_kwVcNlFn",
        "colab_type": "code",
        "outputId": "d4d7eaa3-3908-415a-cc85-223f2b6eba87",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 55
        }
      },
      "cell_type": "code",
      "source": [
        "print(labels)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "[[1, 0], [0, 1], [1, 0], [1, 0], [1, 0], [1, 0], [1, 0], [0, 1], [0, 1], [1, 0], [1, 0], [0, 1], [0, 1], [1, 0], [0, 1], [0, 1], [0, 1], [1, 0], [1, 0], [1, 0], [0, 1], [1, 0], [1, 0], [1, 0], [1, 0], [0, 1], [0, 1], [0, 1], [0, 1], [1, 0], [0, 1], [1, 0], [0, 1], [0, 1], [1, 0], [0, 1], [0, 1], [0, 1], [1, 0], [0, 1], [0, 1], [1, 0], [1, 0], [1, 0], [0, 1], [0, 1], [1, 0], [1, 0], [0, 1], [1, 0], [1, 0], [0, 1], [0, 1], [0, 1], [0, 1], [0, 1], [1, 0], [1, 0], [0, 1], [1, 0], [1, 0], [1, 0], [1, 0], [1, 0], [1, 0], [0, 1], [0, 1], [0, 1], [1, 0], [0, 1], [1, 0], [0, 1], [0, 1], [0, 1], [0, 1], [1, 0], [1, 0], [1, 0], [1, 0], [1, 0], [1, 0], [0, 1], [0, 1], [0, 1], [0, 1], [1, 0], [0, 1], [1, 0], [0, 1], [1, 0], [1, 0], [0, 1], [1, 0], [1, 0], [0, 1], [1, 0], [1, 0], [0, 1], [1, 0], [1, 0], [0, 1], [0, 1], [1, 0], [0, 1], [0, 1], [1, 0], [1, 0], [0, 1], [1, 0], [0, 1], [1, 0], [0, 1], [1, 0], [1, 0], [0, 1], [1, 0], [0, 1], [1, 0], [0, 1], [1, 0], [1, 0], [0, 1], [0, 1], [0, 1], [1, 0], [0, 1], [0, 1], [0, 1], [1, 0], [1, 0], [0, 1], [1, 0], [0, 1], [1, 0], [0, 1], [0, 1], [1, 0], [1, 0], [0, 1], [1, 0], [1, 0], [0, 1], [0, 1], [1, 0], [0, 1], [0, 1], [1, 0], [0, 1], [0, 1], [1, 0], [1, 0], [1, 0], [0, 1], [1, 0], [1, 0], [1, 0], [0, 1], [1, 0], [0, 1], [0, 1], [1, 0], [1, 0], [1, 0], [1, 0], [0, 1], [0, 1], [0, 1], [0, 1], [1, 0], [1, 0], [1, 0], [1, 0], [1, 0], [1, 0], [1, 0], [0, 1], [1, 0], [0, 1], [1, 0], [1, 0], [0, 1], [0, 1], [0, 1], [0, 1], [1, 0], [1, 0], [0, 1], [1, 0], [1, 0], [0, 1], [1, 0], [1, 0], [0, 1], [1, 0], [1, 0], [1, 0], [1, 0], [0, 1], [1, 0], [0, 1], [1, 0], [1, 0], [0, 1], [0, 1], [0, 1], [0, 1], [1, 0], [1, 0], [1, 0], [1, 0], [0, 1], [0, 1], [1, 0], [1, 0], [1, 0], [0, 1], [1, 0], [1, 0], [0, 1], [0, 1], [0, 1], [1, 0], [1, 0], [0, 1], [0, 1], [0, 1], [0, 1], [1, 0], [1, 0], [1, 0], [1, 0], [1, 0], [1, 0], [1, 0], [1, 0], [1, 0], [0, 1], [1, 0], [1, 0], [0, 1], [0, 1], [0, 1], [0, 1], [1, 0], [0, 1], [0, 1], [1, 0], [0, 1], [0, 1], [0, 1], [0, 1], [0, 1], [1, 0], [1, 0], [0, 1], [1, 0], [0, 1], [0, 1], [1, 0], [1, 0], [0, 1], [1, 0], [0, 1], [1, 0], [0, 1], [0, 1], [0, 1], [1, 0], [0, 1], [0, 1], [0, 1], [0, 1], [0, 1], [1, 0], [0, 1], [0, 1], [1, 0], [1, 0], [0, 1], [0, 1], [0, 1], [0, 1], [0, 1], [0, 1], [1, 0], [1, 0], [0, 1], [0, 1], [1, 0], [1, 0], [1, 0], [1, 0], [1, 0], [1, 0], [0, 1], [0, 1], [0, 1], [0, 1], [1, 0], [0, 1], [0, 1], [1, 0], [0, 1], [1, 0], [1, 0], [1, 0], [0, 1], [1, 0], [0, 1], [0, 1], [0, 1], [1, 0], [0, 1], [0, 1], [0, 1], [1, 0], [0, 1], [0, 1], [1, 0], [1, 0], [1, 0], [0, 1], [1, 0], [0, 1], [0, 1], [0, 1], [1, 0], [0, 1], [0, 1], [0, 1], [1, 0], [1, 0], [0, 1], [0, 1], [1, 0], [0, 1], [0, 1], [1, 0], [1, 0], [1, 0], [0, 1], [1, 0], [0, 1], [1, 0], [0, 1], [1, 0], [0, 1], [0, 1], [1, 0], [1, 0], [0, 1], [1, 0], [0, 1], [1, 0], [0, 1], [0, 1], [1, 0], [1, 0], [1, 0], [0, 1], [0, 1], [1, 0], [1, 0], [1, 0], [0, 1], [1, 0], [1, 0], [1, 0], [1, 0], [0, 1], [0, 1], [1, 0], [0, 1], [1, 0], [0, 1], [1, 0], [0, 1], [0, 1], [0, 1], [1, 0], [1, 0], [0, 1], [0, 1], [1, 0], [1, 0], [1, 0], [0, 1], [1, 0], [0, 1], [1, 0], [0, 1], [0, 1], [0, 1], [1, 0], [1, 0], [1, 0], [1, 0], [1, 0], [0, 1], [1, 0], [1, 0], [1, 0], [1, 0], [1, 0], [0, 1], [0, 1], [1, 0], [0, 1], [0, 1], [0, 1], [1, 0], [1, 0], [1, 0], [1, 0], [1, 0], [0, 1], [1, 0], [1, 0], [1, 0], [1, 0], [1, 0], [1, 0], [0, 1], [1, 0], [1, 0], [0, 1], [0, 1], [1, 0], [0, 1], [0, 1], [1, 0], [0, 1], [0, 1], [1, 0], [0, 1], [1, 0], [1, 0], [1, 0], [1, 0], [1, 0], [0, 1], [0, 1], [1, 0], [0, 1], [1, 0], [0, 1], [1, 0], [1, 0], [1, 0], [1, 0], [0, 1], [0, 1], [1, 0], [1, 0], [1, 0], [1, 0], [0, 1], [0, 1], [0, 1], [0, 1], [0, 1], [1, 0], [1, 0], [0, 1], [0, 1], [1, 0], [0, 1], [1, 0], [1, 0], [0, 1], [1, 0], [1, 0], [1, 0], [0, 1], [1, 0], [1, 0], [0, 1], [0, 1], [0, 1], [0, 1], [0, 1], [0, 1], [0, 1], [0, 1], [0, 1], [0, 1], [0, 1], [0, 1], [0, 1], [0, 1], [0, 1], [0, 1], [0, 1], [1, 0], [1, 0], [1, 0], [1, 0], [1, 0], [1, 0], [1, 0], [1, 0], [1, 0], [1, 0], [1, 0], [1, 0], [1, 0], [0, 1], [0, 1], [0, 1], [1, 0], [0, 1], [1, 0], [1, 0], [0, 1], [1, 0], [0, 1], [1, 0], [1, 0], [0, 1], [1, 0], [1, 0], [1, 0], [1, 0], [1, 0], [1, 0], [1, 0], [1, 0], [1, 0], [0, 1], [1, 0], [0, 1], [1, 0], [1, 0], [0, 1], [1, 0], [1, 0], [0, 1], [1, 0], [1, 0], [1, 0], [1, 0], [0, 1], [1, 0], [0, 1], [1, 0], [1, 0], [0, 1], [0, 1], [0, 1], [1, 0], [1, 0], [0, 1], [1, 0], [0, 1], [0, 1], [0, 1], [0, 1], [1, 0], [0, 1], [0, 1], [0, 1], [1, 0], [1, 0], [0, 1], [1, 0], [0, 1], [1, 0], [0, 1], [0, 1], [1, 0], [1, 0], [0, 1], [1, 0], [1, 0], [1, 0], [1, 0], [1, 0], [0, 1], [0, 1], [1, 0], [0, 1], [0, 1], [1, 0], [0, 1], [1, 0], [1, 0], [0, 1], [0, 1], [0, 1], [0, 1], [0, 1], [0, 1], [1, 0], [1, 0], [1, 0], [1, 0], [1, 0], [1, 0], [0, 1], [0, 1], [0, 1], [0, 1], [1, 0], [0, 1], [1, 0], [0, 1], [0, 1], [1, 0], [1, 0], [0, 1], [1, 0], [0, 1], [1, 0], [1, 0], [1, 0], [1, 0], [1, 0], [1, 0], [1, 0], [1, 0], [0, 1], [0, 1], [0, 1], [1, 0], [1, 0], [0, 1], [1, 0], [1, 0], [0, 1], [0, 1], [0, 1], [0, 1], [1, 0], [1, 0], [0, 1], [1, 0], [0, 1], [0, 1], [1, 0], [1, 0], [1, 0], [1, 0], [0, 1], [0, 1], [0, 1], [0, 1], [0, 1], [0, 1], [0, 1], [0, 1], [1, 0], [1, 0], [1, 0], [1, 0], [1, 0], [0, 1], [1, 0], [0, 1], [1, 0], [0, 1], [0, 1], [1, 0], [1, 0], [1, 0], [1, 0], [1, 0], [0, 1], [1, 0], [1, 0], [1, 0], [1, 0], [1, 0], [0, 1], [0, 1], [1, 0], [0, 1], [1, 0], [0, 1], [1, 0], [0, 1], [0, 1], [0, 1], [1, 0], [0, 1], [1, 0], [1, 0], [0, 1], [0, 1], [1, 0], [0, 1], [1, 0], [1, 0], [1, 0], [0, 1], [0, 1], [0, 1], [0, 1], [1, 0], [1, 0], [1, 0], [1, 0], [0, 1], [1, 0], [1, 0], [1, 0], [1, 0], [0, 1], [1, 0], [1, 0], [0, 1], [0, 1], [1, 0], [0, 1], [1, 0], [1, 0], [0, 1], [1, 0], [1, 0], [0, 1], [1, 0], [1, 0], [0, 1], [0, 1], [1, 0], [0, 1], [0, 1], [0, 1], [1, 0], [1, 0], [0, 1], [0, 1], [1, 0], [1, 0], [0, 1], [0, 1], [0, 1], [0, 1], [0, 1], [1, 0], [1, 0], [1, 0], [0, 1], [1, 0], [1, 0], [0, 1], [0, 1], [1, 0], [0, 1], [1, 0], [0, 1], [0, 1], [0, 1], [0, 1], [0, 1], [0, 1], [1, 0], [0, 1], [0, 1], [1, 0], [1, 0], [1, 0], [0, 1], [0, 1], [0, 1], [1, 0], [1, 0], [0, 1], [0, 1], [1, 0], [0, 1], [1, 0], [1, 0], [0, 1], [0, 1], [0, 1], [1, 0], [0, 1], [1, 0], [1, 0], [0, 1], [0, 1], [0, 1], [1, 0], [1, 0], [1, 0], [0, 1], [1, 0], [1, 0], [0, 1], [1, 0], [0, 1], [0, 1], [1, 0], [0, 1], [0, 1], [1, 0], [0, 1], [1, 0], [0, 1], [1, 0], [0, 1], [1, 0], [0, 1], [1, 0], [0, 1], [1, 0], [0, 1], [1, 0], [0, 1], [0, 1], [0, 1], [0, 1], [0, 1], [1, 0], [0, 1], [1, 0], [1, 0], [0, 1], [1, 0], [1, 0], [0, 1], [1, 0], [1, 0], [1, 0], [0, 1], [0, 1], [0, 1], [0, 1], [1, 0], [0, 1], [0, 1], [1, 0], [1, 0], [0, 1], [0, 1], [1, 0], [1, 0], [1, 0], [1, 0], [1, 0], [1, 0], [0, 1], [0, 1], [1, 0], [0, 1], [0, 1], [1, 0], [1, 0], [0, 1], [1, 0], [0, 1], [0, 1], [1, 0], [1, 0], [0, 1], [1, 0], [0, 1], [1, 0], [1, 0], [0, 1], [1, 0], [1, 0], [1, 0], [0, 1], [0, 1], [0, 1], [0, 1], [0, 1], [1, 0], [0, 1], [1, 0], [0, 1], [0, 1], [1, 0], [1, 0], [1, 0], [1, 0], [1, 0], [1, 0], [1, 0], [0, 1], [1, 0], [0, 1], [1, 0], [1, 0], [0, 1], [1, 0], [1, 0], [0, 1], [1, 0], [0, 1], [1, 0], [0, 1], [0, 1], [1, 0], [1, 0], [1, 0], [1, 0], [0, 1], [0, 1], [0, 1], [1, 0], [0, 1], [1, 0], [1, 0], [0, 1], [1, 0], [1, 0], [1, 0], [0, 1], [0, 1], [0, 1], [1, 0], [0, 1], [0, 1], [0, 1], [1, 0], [0, 1], [1, 0], [0, 1], [1, 0], [0, 1], [0, 1], [1, 0], [0, 1], [1, 0], [1, 0], [1, 0], [1, 0], [0, 1], [0, 1], [0, 1], [0, 1], [1, 0], [1, 0], [1, 0], [1, 0], [1, 0], [1, 0], [1, 0], [0, 1], [0, 1], [0, 1], [0, 1], [0, 1], [0, 1], [1, 0], [0, 1], [0, 1], [0, 1], [1, 0], [0, 1], [1, 0], [0, 1], [1, 0], [0, 1], [0, 1], [1, 0], [0, 1], [0, 1], [0, 1], [0, 1], [1, 0], [0, 1], [1, 0], [0, 1], [1, 0], [0, 1], [1, 0], [1, 0], [1, 0], [1, 0], [1, 0], [1, 0], [1, 0], [0, 1], [1, 0], [0, 1], [1, 0], [1, 0], [1, 0], [1, 0], [1, 0], [1, 0], [0, 1], [1, 0], [1, 0], [1, 0], [1, 0], [0, 1], [0, 1], [0, 1], [1, 0], [1, 0], [1, 0], [1, 0], [1, 0], [1, 0], [1, 0], [0, 1]]\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "id": "USAk0IcMNJEC",
        "colab_type": "code",
        "outputId": "5475ff63-264c-44e8-f003-7c6357a1ee71",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        }
      },
      "cell_type": "code",
      "source": [
        "print(len(labels))"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "1000\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "id": "OJLRt6vWJ-tq",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "# random distortion -> Distort"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "HXPDwkBVJ_Kq",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "x1.extend(x1_p)\n",
        "x2.extend(x2_p)\n",
        "x3.extend(x3_p)\n",
        "y.extend(y_p)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "AbF4_MgrYP2r",
        "colab_type": "code",
        "outputId": "f8316478-4cc4-4fa5-f7dd-8699d241c17a",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 90
        }
      },
      "cell_type": "code",
      "source": [
        "print(len(x1))\n",
        "print(len(x2))\n",
        "print(len(x3))\n",
        "print(len(y))"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "1140\n",
            "1140\n",
            "1140\n",
            "1140\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "id": "ygm2EQtrb5Lk",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "import random\n",
        "random.seed(100)\n",
        "zz = list(zip(x1, x2, x3, y))\n",
        "random.shuffle(zz)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "B6Rcj8w5pYPH",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "del x1, x2, x3"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "-42Lqwxucgr8",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "z1 = [a[0] for a in zz]\n",
        "z2 = [a[1] for a in zz]\n",
        "z3 = [a[2] for a in zz]\n",
        "zy = [a[3] for a in zz]"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "nXPcZ7IddM8t",
        "colab_type": "code",
        "outputId": "449b48df-cb17-4216-8213-01ea3fa9438a",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        }
      },
      "cell_type": "code",
      "source": [
        "split = int(0.8*(len(z1)))\n",
        "print(split)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "912\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "id": "I288ceNxc5Im",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "x1_tr = z1[:split]\n",
        "x2_tr = z2[:split]\n",
        "x3_tr = z3[:split]\n",
        "y_tr = zy[:split]\n",
        "\n",
        "x1_ts = z1[split:]\n",
        "x2_ts = z2[split:]\n",
        "x3_ts = z3[split:]\n",
        "y_ts = zy[split:]"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "cOFtSuB0duVF",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "import numpy as np\n",
        "x1_tr = np.array(x1_tr, dtype = 'float32')\n",
        "x2_tr = np.array(x2_tr, dtype = 'float32')\n",
        "x3_tr = np.array(x3_tr, dtype = 'float32')\n",
        "y_tr = np.array(y_tr, dtype = 'float32')\n",
        "y_ts = np.array(y_ts, dtype = 'float32')"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "I3wmWP7seF1e",
        "colab_type": "code",
        "outputId": "29c44b67-6c09-4b3d-c651-a670183c9a76",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        }
      },
      "cell_type": "code",
      "source": [
        "print(np.mean(x1_tr))"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "133.09741\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "id": "L6bVtvHteJPF",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "x1_tr = x1_tr/255.\n",
        "x2_tr = x2_tr/255.\n",
        "x3_tr = x3_tr/255."
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "bgycKN2FSpYm",
        "colab_type": "code",
        "outputId": "510d0272-4756-4f89-f9fe-70267aacb379",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 53
        }
      },
      "cell_type": "code",
      "source": [
        "import keras.backend as K\n",
        "K.image_dim_ordering()"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Using TensorFlow backend.\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "'tf'"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 31
        }
      ]
    },
    {
      "metadata": {
        "id": "ZBn3DM3LbT0K",
        "colab_type": "code",
        "outputId": "b85870b7-cec5-48a6-dc74-d62219175f29",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        }
      },
      "cell_type": "code",
      "source": [
        "np.mean(x1_tr)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "0.52194977"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 32
        }
      ]
    },
    {
      "metadata": {
        "id": "YqezmcFKxuHc",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "cardinality = 1\n",
        "def add_common_layers(y):\n",
        "        y = layers.BatchNormalization()(y)\n",
        "        y = layers.LeakyReLU()(y)\n",
        "\n",
        "        return y\n",
        "\n",
        "def grouped_convolution(y, nb_channels, _strides):\n",
        "    # when `cardinality` == 1 this is just a standard convolution\n",
        "    if cardinality == 1:\n",
        "        return layers.Conv2D(nb_channels, kernel_size=(3, 3), strides=_strides, padding='same')(y)\n",
        "\n",
        "    assert not nb_channels % cardinality\n",
        "    _d = nb_channels // cardinality\n",
        "\n",
        "    # in a grouped convolution layer, input and output channels are divided into `cardinality` groups,\n",
        "    # and convolutions are separately performed within each group\n",
        "    groups = []\n",
        "    for j in range(cardinality):\n",
        "        group = layers.Lambda(lambda z: z[:, :, :, j * _d:j * _d + _d])(y)\n",
        "        groups.append(layers.Conv2D(_d, kernel_size=(3, 3), strides=_strides, padding='same')(group))\n",
        "\n",
        "    # the grouped convolutional layer concatenates them as the outputs of the layer\n",
        "    y = layers.concatenate(groups)\n",
        "\n",
        "    return y\n",
        "\n",
        "def residual_block(y, nb_channels_in, nb_channels_out, _strides=(1, 1), _project_shortcut=False):\n",
        "    \"\"\"\n",
        "    Our network consists of a stack of residual blocks. These blocks have the same topology,\n",
        "    and are subject to two simple rules:\n",
        "    - If producing spatial maps of the same size, the blocks share the same hyper-parameters (width and filter sizes).\n",
        "    - Each time the spatial map is down-sampled by a factor of 2, the width of the blocks is multiplied by a factor of 2.\n",
        "    \"\"\"\n",
        "    shortcut = y\n",
        "\n",
        "    # we modify the residual building block as a bottleneck design to make the network more economical\n",
        "    y = layers.Conv2D(nb_channels_in, kernel_size=(1, 1), strides=(1, 1), padding='same')(y)\n",
        "    y = add_common_layers(y)\n",
        "\n",
        "    # ResNeXt (identical to ResNet when `cardinality` == 1)\n",
        "    y = grouped_convolution(y, nb_channels_in, _strides=_strides)\n",
        "    y = add_common_layers(y)\n",
        "\n",
        "    y = layers.Conv2D(nb_channels_out, kernel_size=(1, 1), strides=(1, 1), padding='same')(y)\n",
        "    # batch normalization is employed after aggregating the transformations and before adding to the shortcut\n",
        "    y = layers.BatchNormalization()(y)\n",
        "\n",
        "    # identity shortcuts used directly when the input and output are of the same dimensions\n",
        "    if _project_shortcut or _strides != (1, 1):\n",
        "        # when the dimensions increase projection shortcut is used to match dimensions (done by 1×1 convolutions)\n",
        "        # when the shortcuts go across feature maps of two sizes, they are performed with a stride of 2\n",
        "        shortcut = layers.Conv2D(nb_channels_out, kernel_size=(1, 1), strides=_strides, padding='same')(shortcut)\n",
        "        shortcut = layers.BatchNormalization()(shortcut)\n",
        "\n",
        "    y = layers.add([shortcut, y])\n",
        "\n",
        "    # relu is performed right after each batch normalization,\n",
        "    # expect for the output of the block where relu is performed after the adding to the shortcut\n",
        "    y = layers.LeakyReLU()(y)\n",
        "\n",
        "    return y"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "fjbxqYl8ToKN",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "from keras.layers import Conv2D, MaxPooling2D, Input, Concatenate, Flatten, Dense, BatchNormalization, Activation, SpatialDropout2D\n",
        "import keras\n",
        "from keras import Model"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "Q70maxojyvuf",
        "colab_type": "code",
        "outputId": "5cd001d8-8350-48c3-d4fc-71f3051d2cc9",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 3639
        }
      },
      "cell_type": "code",
      "source": [
        "from keras import layers\n",
        "def multi_resnet(): \n",
        "  \n",
        "    input_img1 = Input(shape=(224, 224, 3))\n",
        "    input_img2 = Input(shape=(224, 224, 3))\n",
        "    input_img3 = Input(shape=(224, 224, 3))\n",
        "\n",
        "    cm1 = Conv2D(64, (3, 3), padding='same', activation='relu')(input_img1)\n",
        "    cm1 = Conv2D(64, (3, 3), padding='same', activation='relu')(cm1)\n",
        "\n",
        "    cm2 = Conv2D(64, (3, 3), padding='same', activation='relu')(input_img2)\n",
        "    cm2 = Conv2D(64, (3, 3), padding='same', activation='relu')(cm2)\n",
        "\n",
        "\n",
        "    cm3 = Conv2D(64, (3, 3), padding='same',  activation='relu')(input_img3)\n",
        "    cm3 = Conv2D(64, (3, 3), padding='same',  activation='relu')(cm3)\n",
        "\n",
        "    merge_1 = keras.layers.concatenate([cm1, cm2, cm3], axis=1)\n",
        "    \n",
        "    \n",
        "    # conv1\n",
        "    x = layers.Conv2D(64, kernel_size=(7, 7), strides=(2, 2), padding='same')(merge_1)\n",
        "    x = add_common_layers(x)\n",
        "\n",
        "    # conv2\n",
        "    x = layers.MaxPool2D(pool_size=(3, 3), strides=(2, 2), padding='same')(x)\n",
        "    for i in range(3):\n",
        "        project_shortcut = True if i == 0 else False\n",
        "        x = residual_block(x, 128, 256, _project_shortcut=project_shortcut)\n",
        "\n",
        "    # conv3\n",
        "    for i in range(4):\n",
        "    #    # down-sampling is performed by conv3_1, conv4_1, and conv5_1 with a stride of 2\n",
        "        strides = (2, 2) if i == 0 else (1, 1)\n",
        "        x = residual_block(x, 256, 512, _strides=strides)\n",
        "\n",
        "    # conv4\n",
        "    #for i in range(6):\n",
        "    #    strides = (2, 2) if i == 0 else (1, 1)\n",
        "    #    x = residual_block(x, 512, 1024, _strides=strides)\n",
        "\n",
        "    # conv5\n",
        "    #for i in range(3):\n",
        "    #    strides = (2, 2) if i == 0 else (1, 1)\n",
        "    #    x = residual_block(x, 1024, 2048, _strides=strides)\n",
        "\n",
        "    x = layers.GlobalAveragePooling2D()(x)\n",
        "    #flat = Flatten()(x)\n",
        "\n",
        " \n",
        "    hidden1 = Dense(512, activation='relu')(x)\n",
        "    output = Dense(2, activation='sigmoid')(hidden1)\n",
        "    model = Model(inputs=[input_img1, input_img2, input_img3], outputs=output)\n",
        "\n",
        "    return model\n",
        "model = multi_resnet()\n",
        "print(model.summary())"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "__________________________________________________________________________________________________\n",
            "Layer (type)                    Output Shape         Param #     Connected to                     \n",
            "==================================================================================================\n",
            "input_1 (InputLayer)            (None, 224, 224, 3)  0                                            \n",
            "__________________________________________________________________________________________________\n",
            "input_2 (InputLayer)            (None, 224, 224, 3)  0                                            \n",
            "__________________________________________________________________________________________________\n",
            "input_3 (InputLayer)            (None, 224, 224, 3)  0                                            \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_1 (Conv2D)               (None, 224, 224, 64) 1792        input_1[0][0]                    \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_3 (Conv2D)               (None, 224, 224, 64) 1792        input_2[0][0]                    \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_5 (Conv2D)               (None, 224, 224, 64) 1792        input_3[0][0]                    \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_2 (Conv2D)               (None, 224, 224, 64) 36928       conv2d_1[0][0]                   \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_4 (Conv2D)               (None, 224, 224, 64) 36928       conv2d_3[0][0]                   \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_6 (Conv2D)               (None, 224, 224, 64) 36928       conv2d_5[0][0]                   \n",
            "__________________________________________________________________________________________________\n",
            "concatenate_1 (Concatenate)     (None, 672, 224, 64) 0           conv2d_2[0][0]                   \n",
            "                                                                 conv2d_4[0][0]                   \n",
            "                                                                 conv2d_6[0][0]                   \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_7 (Conv2D)               (None, 336, 112, 64) 200768      concatenate_1[0][0]              \n",
            "__________________________________________________________________________________________________\n",
            "batch_normalization_1 (BatchNor (None, 336, 112, 64) 256         conv2d_7[0][0]                   \n",
            "__________________________________________________________________________________________________\n",
            "leaky_re_lu_1 (LeakyReLU)       (None, 336, 112, 64) 0           batch_normalization_1[0][0]      \n",
            "__________________________________________________________________________________________________\n",
            "max_pooling2d_1 (MaxPooling2D)  (None, 168, 56, 64)  0           leaky_re_lu_1[0][0]              \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_8 (Conv2D)               (None, 168, 56, 128) 8320        max_pooling2d_1[0][0]            \n",
            "__________________________________________________________________________________________________\n",
            "batch_normalization_2 (BatchNor (None, 168, 56, 128) 512         conv2d_8[0][0]                   \n",
            "__________________________________________________________________________________________________\n",
            "leaky_re_lu_2 (LeakyReLU)       (None, 168, 56, 128) 0           batch_normalization_2[0][0]      \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_9 (Conv2D)               (None, 168, 56, 128) 147584      leaky_re_lu_2[0][0]              \n",
            "__________________________________________________________________________________________________\n",
            "batch_normalization_3 (BatchNor (None, 168, 56, 128) 512         conv2d_9[0][0]                   \n",
            "__________________________________________________________________________________________________\n",
            "leaky_re_lu_3 (LeakyReLU)       (None, 168, 56, 128) 0           batch_normalization_3[0][0]      \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_11 (Conv2D)              (None, 168, 56, 256) 16640       max_pooling2d_1[0][0]            \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_10 (Conv2D)              (None, 168, 56, 256) 33024       leaky_re_lu_3[0][0]              \n",
            "__________________________________________________________________________________________________\n",
            "batch_normalization_5 (BatchNor (None, 168, 56, 256) 1024        conv2d_11[0][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "batch_normalization_4 (BatchNor (None, 168, 56, 256) 1024        conv2d_10[0][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "add_1 (Add)                     (None, 168, 56, 256) 0           batch_normalization_5[0][0]      \n",
            "                                                                 batch_normalization_4[0][0]      \n",
            "__________________________________________________________________________________________________\n",
            "leaky_re_lu_4 (LeakyReLU)       (None, 168, 56, 256) 0           add_1[0][0]                      \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_12 (Conv2D)              (None, 168, 56, 128) 32896       leaky_re_lu_4[0][0]              \n",
            "__________________________________________________________________________________________________\n",
            "batch_normalization_6 (BatchNor (None, 168, 56, 128) 512         conv2d_12[0][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "leaky_re_lu_5 (LeakyReLU)       (None, 168, 56, 128) 0           batch_normalization_6[0][0]      \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_13 (Conv2D)              (None, 168, 56, 128) 147584      leaky_re_lu_5[0][0]              \n",
            "__________________________________________________________________________________________________\n",
            "batch_normalization_7 (BatchNor (None, 168, 56, 128) 512         conv2d_13[0][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "leaky_re_lu_6 (LeakyReLU)       (None, 168, 56, 128) 0           batch_normalization_7[0][0]      \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_14 (Conv2D)              (None, 168, 56, 256) 33024       leaky_re_lu_6[0][0]              \n",
            "__________________________________________________________________________________________________\n",
            "batch_normalization_8 (BatchNor (None, 168, 56, 256) 1024        conv2d_14[0][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "add_2 (Add)                     (None, 168, 56, 256) 0           leaky_re_lu_4[0][0]              \n",
            "                                                                 batch_normalization_8[0][0]      \n",
            "__________________________________________________________________________________________________\n",
            "leaky_re_lu_7 (LeakyReLU)       (None, 168, 56, 256) 0           add_2[0][0]                      \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_15 (Conv2D)              (None, 168, 56, 128) 32896       leaky_re_lu_7[0][0]              \n",
            "__________________________________________________________________________________________________\n",
            "batch_normalization_9 (BatchNor (None, 168, 56, 128) 512         conv2d_15[0][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "leaky_re_lu_8 (LeakyReLU)       (None, 168, 56, 128) 0           batch_normalization_9[0][0]      \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_16 (Conv2D)              (None, 168, 56, 128) 147584      leaky_re_lu_8[0][0]              \n",
            "__________________________________________________________________________________________________\n",
            "batch_normalization_10 (BatchNo (None, 168, 56, 128) 512         conv2d_16[0][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "leaky_re_lu_9 (LeakyReLU)       (None, 168, 56, 128) 0           batch_normalization_10[0][0]     \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_17 (Conv2D)              (None, 168, 56, 256) 33024       leaky_re_lu_9[0][0]              \n",
            "__________________________________________________________________________________________________\n",
            "batch_normalization_11 (BatchNo (None, 168, 56, 256) 1024        conv2d_17[0][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "add_3 (Add)                     (None, 168, 56, 256) 0           leaky_re_lu_7[0][0]              \n",
            "                                                                 batch_normalization_11[0][0]     \n",
            "__________________________________________________________________________________________________\n",
            "leaky_re_lu_10 (LeakyReLU)      (None, 168, 56, 256) 0           add_3[0][0]                      \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_18 (Conv2D)              (None, 168, 56, 256) 65792       leaky_re_lu_10[0][0]             \n",
            "__________________________________________________________________________________________________\n",
            "batch_normalization_12 (BatchNo (None, 168, 56, 256) 1024        conv2d_18[0][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "leaky_re_lu_11 (LeakyReLU)      (None, 168, 56, 256) 0           batch_normalization_12[0][0]     \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_19 (Conv2D)              (None, 84, 28, 256)  590080      leaky_re_lu_11[0][0]             \n",
            "__________________________________________________________________________________________________\n",
            "batch_normalization_13 (BatchNo (None, 84, 28, 256)  1024        conv2d_19[0][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "leaky_re_lu_12 (LeakyReLU)      (None, 84, 28, 256)  0           batch_normalization_13[0][0]     \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_21 (Conv2D)              (None, 84, 28, 512)  131584      leaky_re_lu_10[0][0]             \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_20 (Conv2D)              (None, 84, 28, 512)  131584      leaky_re_lu_12[0][0]             \n",
            "__________________________________________________________________________________________________\n",
            "batch_normalization_15 (BatchNo (None, 84, 28, 512)  2048        conv2d_21[0][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "batch_normalization_14 (BatchNo (None, 84, 28, 512)  2048        conv2d_20[0][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "add_4 (Add)                     (None, 84, 28, 512)  0           batch_normalization_15[0][0]     \n",
            "                                                                 batch_normalization_14[0][0]     \n",
            "__________________________________________________________________________________________________\n",
            "leaky_re_lu_13 (LeakyReLU)      (None, 84, 28, 512)  0           add_4[0][0]                      \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_22 (Conv2D)              (None, 84, 28, 256)  131328      leaky_re_lu_13[0][0]             \n",
            "__________________________________________________________________________________________________\n",
            "batch_normalization_16 (BatchNo (None, 84, 28, 256)  1024        conv2d_22[0][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "leaky_re_lu_14 (LeakyReLU)      (None, 84, 28, 256)  0           batch_normalization_16[0][0]     \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_23 (Conv2D)              (None, 84, 28, 256)  590080      leaky_re_lu_14[0][0]             \n",
            "__________________________________________________________________________________________________\n",
            "batch_normalization_17 (BatchNo (None, 84, 28, 256)  1024        conv2d_23[0][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "leaky_re_lu_15 (LeakyReLU)      (None, 84, 28, 256)  0           batch_normalization_17[0][0]     \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_24 (Conv2D)              (None, 84, 28, 512)  131584      leaky_re_lu_15[0][0]             \n",
            "__________________________________________________________________________________________________\n",
            "batch_normalization_18 (BatchNo (None, 84, 28, 512)  2048        conv2d_24[0][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "add_5 (Add)                     (None, 84, 28, 512)  0           leaky_re_lu_13[0][0]             \n",
            "                                                                 batch_normalization_18[0][0]     \n",
            "__________________________________________________________________________________________________\n",
            "leaky_re_lu_16 (LeakyReLU)      (None, 84, 28, 512)  0           add_5[0][0]                      \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_25 (Conv2D)              (None, 84, 28, 256)  131328      leaky_re_lu_16[0][0]             \n",
            "__________________________________________________________________________________________________\n",
            "batch_normalization_19 (BatchNo (None, 84, 28, 256)  1024        conv2d_25[0][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "leaky_re_lu_17 (LeakyReLU)      (None, 84, 28, 256)  0           batch_normalization_19[0][0]     \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_26 (Conv2D)              (None, 84, 28, 256)  590080      leaky_re_lu_17[0][0]             \n",
            "__________________________________________________________________________________________________\n",
            "batch_normalization_20 (BatchNo (None, 84, 28, 256)  1024        conv2d_26[0][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "leaky_re_lu_18 (LeakyReLU)      (None, 84, 28, 256)  0           batch_normalization_20[0][0]     \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_27 (Conv2D)              (None, 84, 28, 512)  131584      leaky_re_lu_18[0][0]             \n",
            "__________________________________________________________________________________________________\n",
            "batch_normalization_21 (BatchNo (None, 84, 28, 512)  2048        conv2d_27[0][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "add_6 (Add)                     (None, 84, 28, 512)  0           leaky_re_lu_16[0][0]             \n",
            "                                                                 batch_normalization_21[0][0]     \n",
            "__________________________________________________________________________________________________\n",
            "leaky_re_lu_19 (LeakyReLU)      (None, 84, 28, 512)  0           add_6[0][0]                      \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_28 (Conv2D)              (None, 84, 28, 256)  131328      leaky_re_lu_19[0][0]             \n",
            "__________________________________________________________________________________________________\n",
            "batch_normalization_22 (BatchNo (None, 84, 28, 256)  1024        conv2d_28[0][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "leaky_re_lu_20 (LeakyReLU)      (None, 84, 28, 256)  0           batch_normalization_22[0][0]     \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_29 (Conv2D)              (None, 84, 28, 256)  590080      leaky_re_lu_20[0][0]             \n",
            "__________________________________________________________________________________________________\n",
            "batch_normalization_23 (BatchNo (None, 84, 28, 256)  1024        conv2d_29[0][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "leaky_re_lu_21 (LeakyReLU)      (None, 84, 28, 256)  0           batch_normalization_23[0][0]     \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_30 (Conv2D)              (None, 84, 28, 512)  131584      leaky_re_lu_21[0][0]             \n",
            "__________________________________________________________________________________________________\n",
            "batch_normalization_24 (BatchNo (None, 84, 28, 512)  2048        conv2d_30[0][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "add_7 (Add)                     (None, 84, 28, 512)  0           leaky_re_lu_19[0][0]             \n",
            "                                                                 batch_normalization_24[0][0]     \n",
            "__________________________________________________________________________________________________\n",
            "leaky_re_lu_22 (LeakyReLU)      (None, 84, 28, 512)  0           add_7[0][0]                      \n",
            "__________________________________________________________________________________________________\n",
            "global_average_pooling2d_1 (Glo (None, 512)          0           leaky_re_lu_22[0][0]             \n",
            "__________________________________________________________________________________________________\n",
            "dense_1 (Dense)                 (None, 512)          262656      global_average_pooling2d_1[0][0] \n",
            "__________________________________________________________________________________________________\n",
            "dense_2 (Dense)                 (None, 2)            1026        dense_1[0][0]                    \n",
            "==================================================================================================\n",
            "Total params: 4,717,058\n",
            "Trainable params: 4,704,130\n",
            "Non-trainable params: 12,928\n",
            "__________________________________________________________________________________________________\n",
            "None\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "id": "-k5Jd_PB1v-1",
        "colab_type": "code",
        "outputId": "78e4cca1-1420-4075-b05e-f31ef07a7df6",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 3621
        }
      },
      "cell_type": "code",
      "source": [
        "model.summary()"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "__________________________________________________________________________________________________\n",
            "Layer (type)                    Output Shape         Param #     Connected to                     \n",
            "==================================================================================================\n",
            "input_1 (InputLayer)            (None, 224, 224, 3)  0                                            \n",
            "__________________________________________________________________________________________________\n",
            "input_2 (InputLayer)            (None, 224, 224, 3)  0                                            \n",
            "__________________________________________________________________________________________________\n",
            "input_3 (InputLayer)            (None, 224, 224, 3)  0                                            \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_1 (Conv2D)               (None, 224, 224, 64) 1792        input_1[0][0]                    \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_3 (Conv2D)               (None, 224, 224, 64) 1792        input_2[0][0]                    \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_5 (Conv2D)               (None, 224, 224, 64) 1792        input_3[0][0]                    \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_2 (Conv2D)               (None, 224, 224, 64) 36928       conv2d_1[0][0]                   \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_4 (Conv2D)               (None, 224, 224, 64) 36928       conv2d_3[0][0]                   \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_6 (Conv2D)               (None, 224, 224, 64) 36928       conv2d_5[0][0]                   \n",
            "__________________________________________________________________________________________________\n",
            "concatenate_1 (Concatenate)     (None, 672, 224, 64) 0           conv2d_2[0][0]                   \n",
            "                                                                 conv2d_4[0][0]                   \n",
            "                                                                 conv2d_6[0][0]                   \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_7 (Conv2D)               (None, 336, 112, 64) 200768      concatenate_1[0][0]              \n",
            "__________________________________________________________________________________________________\n",
            "batch_normalization_1 (BatchNor (None, 336, 112, 64) 256         conv2d_7[0][0]                   \n",
            "__________________________________________________________________________________________________\n",
            "leaky_re_lu_1 (LeakyReLU)       (None, 336, 112, 64) 0           batch_normalization_1[0][0]      \n",
            "__________________________________________________________________________________________________\n",
            "max_pooling2d_1 (MaxPooling2D)  (None, 168, 56, 64)  0           leaky_re_lu_1[0][0]              \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_8 (Conv2D)               (None, 168, 56, 128) 8320        max_pooling2d_1[0][0]            \n",
            "__________________________________________________________________________________________________\n",
            "batch_normalization_2 (BatchNor (None, 168, 56, 128) 512         conv2d_8[0][0]                   \n",
            "__________________________________________________________________________________________________\n",
            "leaky_re_lu_2 (LeakyReLU)       (None, 168, 56, 128) 0           batch_normalization_2[0][0]      \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_9 (Conv2D)               (None, 168, 56, 128) 147584      leaky_re_lu_2[0][0]              \n",
            "__________________________________________________________________________________________________\n",
            "batch_normalization_3 (BatchNor (None, 168, 56, 128) 512         conv2d_9[0][0]                   \n",
            "__________________________________________________________________________________________________\n",
            "leaky_re_lu_3 (LeakyReLU)       (None, 168, 56, 128) 0           batch_normalization_3[0][0]      \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_11 (Conv2D)              (None, 168, 56, 256) 16640       max_pooling2d_1[0][0]            \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_10 (Conv2D)              (None, 168, 56, 256) 33024       leaky_re_lu_3[0][0]              \n",
            "__________________________________________________________________________________________________\n",
            "batch_normalization_5 (BatchNor (None, 168, 56, 256) 1024        conv2d_11[0][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "batch_normalization_4 (BatchNor (None, 168, 56, 256) 1024        conv2d_10[0][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "add_1 (Add)                     (None, 168, 56, 256) 0           batch_normalization_5[0][0]      \n",
            "                                                                 batch_normalization_4[0][0]      \n",
            "__________________________________________________________________________________________________\n",
            "leaky_re_lu_4 (LeakyReLU)       (None, 168, 56, 256) 0           add_1[0][0]                      \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_12 (Conv2D)              (None, 168, 56, 128) 32896       leaky_re_lu_4[0][0]              \n",
            "__________________________________________________________________________________________________\n",
            "batch_normalization_6 (BatchNor (None, 168, 56, 128) 512         conv2d_12[0][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "leaky_re_lu_5 (LeakyReLU)       (None, 168, 56, 128) 0           batch_normalization_6[0][0]      \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_13 (Conv2D)              (None, 168, 56, 128) 147584      leaky_re_lu_5[0][0]              \n",
            "__________________________________________________________________________________________________\n",
            "batch_normalization_7 (BatchNor (None, 168, 56, 128) 512         conv2d_13[0][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "leaky_re_lu_6 (LeakyReLU)       (None, 168, 56, 128) 0           batch_normalization_7[0][0]      \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_14 (Conv2D)              (None, 168, 56, 256) 33024       leaky_re_lu_6[0][0]              \n",
            "__________________________________________________________________________________________________\n",
            "batch_normalization_8 (BatchNor (None, 168, 56, 256) 1024        conv2d_14[0][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "add_2 (Add)                     (None, 168, 56, 256) 0           leaky_re_lu_4[0][0]              \n",
            "                                                                 batch_normalization_8[0][0]      \n",
            "__________________________________________________________________________________________________\n",
            "leaky_re_lu_7 (LeakyReLU)       (None, 168, 56, 256) 0           add_2[0][0]                      \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_15 (Conv2D)              (None, 168, 56, 128) 32896       leaky_re_lu_7[0][0]              \n",
            "__________________________________________________________________________________________________\n",
            "batch_normalization_9 (BatchNor (None, 168, 56, 128) 512         conv2d_15[0][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "leaky_re_lu_8 (LeakyReLU)       (None, 168, 56, 128) 0           batch_normalization_9[0][0]      \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_16 (Conv2D)              (None, 168, 56, 128) 147584      leaky_re_lu_8[0][0]              \n",
            "__________________________________________________________________________________________________\n",
            "batch_normalization_10 (BatchNo (None, 168, 56, 128) 512         conv2d_16[0][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "leaky_re_lu_9 (LeakyReLU)       (None, 168, 56, 128) 0           batch_normalization_10[0][0]     \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_17 (Conv2D)              (None, 168, 56, 256) 33024       leaky_re_lu_9[0][0]              \n",
            "__________________________________________________________________________________________________\n",
            "batch_normalization_11 (BatchNo (None, 168, 56, 256) 1024        conv2d_17[0][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "add_3 (Add)                     (None, 168, 56, 256) 0           leaky_re_lu_7[0][0]              \n",
            "                                                                 batch_normalization_11[0][0]     \n",
            "__________________________________________________________________________________________________\n",
            "leaky_re_lu_10 (LeakyReLU)      (None, 168, 56, 256) 0           add_3[0][0]                      \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_18 (Conv2D)              (None, 168, 56, 256) 65792       leaky_re_lu_10[0][0]             \n",
            "__________________________________________________________________________________________________\n",
            "batch_normalization_12 (BatchNo (None, 168, 56, 256) 1024        conv2d_18[0][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "leaky_re_lu_11 (LeakyReLU)      (None, 168, 56, 256) 0           batch_normalization_12[0][0]     \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_19 (Conv2D)              (None, 84, 28, 256)  590080      leaky_re_lu_11[0][0]             \n",
            "__________________________________________________________________________________________________\n",
            "batch_normalization_13 (BatchNo (None, 84, 28, 256)  1024        conv2d_19[0][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "leaky_re_lu_12 (LeakyReLU)      (None, 84, 28, 256)  0           batch_normalization_13[0][0]     \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_21 (Conv2D)              (None, 84, 28, 512)  131584      leaky_re_lu_10[0][0]             \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_20 (Conv2D)              (None, 84, 28, 512)  131584      leaky_re_lu_12[0][0]             \n",
            "__________________________________________________________________________________________________\n",
            "batch_normalization_15 (BatchNo (None, 84, 28, 512)  2048        conv2d_21[0][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "batch_normalization_14 (BatchNo (None, 84, 28, 512)  2048        conv2d_20[0][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "add_4 (Add)                     (None, 84, 28, 512)  0           batch_normalization_15[0][0]     \n",
            "                                                                 batch_normalization_14[0][0]     \n",
            "__________________________________________________________________________________________________\n",
            "leaky_re_lu_13 (LeakyReLU)      (None, 84, 28, 512)  0           add_4[0][0]                      \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_22 (Conv2D)              (None, 84, 28, 256)  131328      leaky_re_lu_13[0][0]             \n",
            "__________________________________________________________________________________________________\n",
            "batch_normalization_16 (BatchNo (None, 84, 28, 256)  1024        conv2d_22[0][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "leaky_re_lu_14 (LeakyReLU)      (None, 84, 28, 256)  0           batch_normalization_16[0][0]     \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_23 (Conv2D)              (None, 84, 28, 256)  590080      leaky_re_lu_14[0][0]             \n",
            "__________________________________________________________________________________________________\n",
            "batch_normalization_17 (BatchNo (None, 84, 28, 256)  1024        conv2d_23[0][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "leaky_re_lu_15 (LeakyReLU)      (None, 84, 28, 256)  0           batch_normalization_17[0][0]     \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_24 (Conv2D)              (None, 84, 28, 512)  131584      leaky_re_lu_15[0][0]             \n",
            "__________________________________________________________________________________________________\n",
            "batch_normalization_18 (BatchNo (None, 84, 28, 512)  2048        conv2d_24[0][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "add_5 (Add)                     (None, 84, 28, 512)  0           leaky_re_lu_13[0][0]             \n",
            "                                                                 batch_normalization_18[0][0]     \n",
            "__________________________________________________________________________________________________\n",
            "leaky_re_lu_16 (LeakyReLU)      (None, 84, 28, 512)  0           add_5[0][0]                      \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_25 (Conv2D)              (None, 84, 28, 256)  131328      leaky_re_lu_16[0][0]             \n",
            "__________________________________________________________________________________________________\n",
            "batch_normalization_19 (BatchNo (None, 84, 28, 256)  1024        conv2d_25[0][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "leaky_re_lu_17 (LeakyReLU)      (None, 84, 28, 256)  0           batch_normalization_19[0][0]     \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_26 (Conv2D)              (None, 84, 28, 256)  590080      leaky_re_lu_17[0][0]             \n",
            "__________________________________________________________________________________________________\n",
            "batch_normalization_20 (BatchNo (None, 84, 28, 256)  1024        conv2d_26[0][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "leaky_re_lu_18 (LeakyReLU)      (None, 84, 28, 256)  0           batch_normalization_20[0][0]     \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_27 (Conv2D)              (None, 84, 28, 512)  131584      leaky_re_lu_18[0][0]             \n",
            "__________________________________________________________________________________________________\n",
            "batch_normalization_21 (BatchNo (None, 84, 28, 512)  2048        conv2d_27[0][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "add_6 (Add)                     (None, 84, 28, 512)  0           leaky_re_lu_16[0][0]             \n",
            "                                                                 batch_normalization_21[0][0]     \n",
            "__________________________________________________________________________________________________\n",
            "leaky_re_lu_19 (LeakyReLU)      (None, 84, 28, 512)  0           add_6[0][0]                      \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_28 (Conv2D)              (None, 84, 28, 256)  131328      leaky_re_lu_19[0][0]             \n",
            "__________________________________________________________________________________________________\n",
            "batch_normalization_22 (BatchNo (None, 84, 28, 256)  1024        conv2d_28[0][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "leaky_re_lu_20 (LeakyReLU)      (None, 84, 28, 256)  0           batch_normalization_22[0][0]     \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_29 (Conv2D)              (None, 84, 28, 256)  590080      leaky_re_lu_20[0][0]             \n",
            "__________________________________________________________________________________________________\n",
            "batch_normalization_23 (BatchNo (None, 84, 28, 256)  1024        conv2d_29[0][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "leaky_re_lu_21 (LeakyReLU)      (None, 84, 28, 256)  0           batch_normalization_23[0][0]     \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_30 (Conv2D)              (None, 84, 28, 512)  131584      leaky_re_lu_21[0][0]             \n",
            "__________________________________________________________________________________________________\n",
            "batch_normalization_24 (BatchNo (None, 84, 28, 512)  2048        conv2d_30[0][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "add_7 (Add)                     (None, 84, 28, 512)  0           leaky_re_lu_19[0][0]             \n",
            "                                                                 batch_normalization_24[0][0]     \n",
            "__________________________________________________________________________________________________\n",
            "leaky_re_lu_22 (LeakyReLU)      (None, 84, 28, 512)  0           add_7[0][0]                      \n",
            "__________________________________________________________________________________________________\n",
            "global_average_pooling2d_1 (Glo (None, 512)          0           leaky_re_lu_22[0][0]             \n",
            "__________________________________________________________________________________________________\n",
            "dense_1 (Dense)                 (None, 512)          262656      global_average_pooling2d_1[0][0] \n",
            "__________________________________________________________________________________________________\n",
            "dense_2 (Dense)                 (None, 2)            1026        dense_1[0][0]                    \n",
            "==================================================================================================\n",
            "Total params: 4,717,058\n",
            "Trainable params: 4,704,130\n",
            "Non-trainable params: 12,928\n",
            "__________________________________________________________________________________________________\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "id": "Kypjt3fKtxQt",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "from keras.layers import Conv2D, MaxPooling2D, Input, Concatenate, Flatten, Dense, BatchNormalization, Activation, SpatialDropout2D\n",
        "import keras\n",
        "from keras import Model\n",
        "\n",
        "dr = 0.1 # dropout\n",
        "drt = 1 # dilation rate\n",
        "\n",
        "input_img1 = Input(shape=(224, 224, 3))\n",
        "input_img2 = Input(shape=(224, 224, 3))\n",
        "input_img3 = Input(shape=(224, 224, 3))\n",
        "\n",
        "cm1 = Conv2D(64, (3, 3), padding='same', activation='relu')(input_img1)\n",
        "cm1 = Conv2D(64, (3, 3), padding='same', activation='relu')(cm1)\n",
        "\n",
        "cm2 = Conv2D(64, (3, 3), padding='same', activation='relu')(input_img2)\n",
        "cm2 = Conv2D(64, (3, 3), padding='same', activation='relu')(cm2)\n",
        "\n",
        "\n",
        "cm3 = Conv2D(64, (3, 3), padding='same',  activation='relu')(input_img3)\n",
        "cm3 = Conv2D(64, (3, 3), padding='same',  activation='relu')(cm3)\n",
        "\n",
        "merge_l = keras.layers.concatenate([cm1, cm2, cm3], axis=1)\n",
        "\n",
        "conv21 = Conv2D(32, kernel_size=3)(merge_l)\n",
        "#conv21 = BatchNormalization(axis=3)(conv21) # tensorflow backend\n",
        "conv21 = Activation('relu')(conv21)\n",
        "conv21 = Conv2D(16, kernel_size=3)(conv21)\n",
        "#conv21 = SpatialDropout2D(dr)(conv21)\n",
        "conv21 = Activation('relu')(conv21)\n",
        "pool21 = MaxPooling2D(pool_size=(2, 2))(conv21)\n",
        "conv22 = Conv2D(32, kernel_size=3, activation='relu')(pool21)\n",
        "pool22 = MaxPooling2D(pool_size=(2, 2))(conv22)\n",
        "conv22 = Conv2D(64, kernel_size=3, activation='relu')(pool21)\n",
        "pool22 = MaxPooling2D(pool_size=(2, 2))(conv22)\n",
        "\n",
        "flat = Flatten()(pool22)\n",
        "\n",
        "\n",
        "hidden1 = Dense(16, activation='relu')(flat)\n",
        "output = Dense(2, activation='softmax')(hidden1)\n",
        "model = Model(inputs=[input_img1, input_img2, input_img3], outputs=output)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "hEaL-Vx5wSO2",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        ""
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "Mu-delH4y3a1",
        "colab_type": "code",
        "outputId": "f7213160-4fc3-4091-c0bd-ca266f87ff18",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 909
        }
      },
      "cell_type": "code",
      "source": [
        "model.summary()"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "__________________________________________________________________________________________________\n",
            "Layer (type)                    Output Shape         Param #     Connected to                     \n",
            "==================================================================================================\n",
            "input_16 (InputLayer)           (None, 224, 224, 3)  0                                            \n",
            "__________________________________________________________________________________________________\n",
            "input_17 (InputLayer)           (None, 224, 224, 3)  0                                            \n",
            "__________________________________________________________________________________________________\n",
            "input_18 (InputLayer)           (None, 224, 224, 3)  0                                            \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_71 (Conv2D)              (None, 224, 224, 64) 1792        input_16[0][0]                   \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_73 (Conv2D)              (None, 224, 224, 64) 1792        input_17[0][0]                   \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_75 (Conv2D)              (None, 224, 224, 64) 1792        input_18[0][0]                   \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_72 (Conv2D)              (None, 224, 224, 64) 36928       conv2d_71[0][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_74 (Conv2D)              (None, 224, 224, 64) 36928       conv2d_73[0][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_76 (Conv2D)              (None, 224, 224, 64) 36928       conv2d_75[0][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "concatenate_6 (Concatenate)     (None, 672, 224, 64) 0           conv2d_72[0][0]                  \n",
            "                                                                 conv2d_74[0][0]                  \n",
            "                                                                 conv2d_76[0][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_77 (Conv2D)              (None, 670, 222, 32) 18464       concatenate_6[0][0]              \n",
            "__________________________________________________________________________________________________\n",
            "activation_9 (Activation)       (None, 670, 222, 32) 0           conv2d_77[0][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_78 (Conv2D)              (None, 668, 220, 16) 4624        activation_9[0][0]               \n",
            "__________________________________________________________________________________________________\n",
            "activation_10 (Activation)      (None, 668, 220, 16) 0           conv2d_78[0][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "max_pooling2d_14 (MaxPooling2D) (None, 334, 110, 16) 0           activation_10[0][0]              \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_80 (Conv2D)              (None, 332, 108, 64) 9280        max_pooling2d_14[0][0]           \n",
            "__________________________________________________________________________________________________\n",
            "max_pooling2d_16 (MaxPooling2D) (None, 166, 54, 64)  0           conv2d_80[0][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "flatten_5 (Flatten)             (None, 573696)       0           max_pooling2d_16[0][0]           \n",
            "__________________________________________________________________________________________________\n",
            "dense_11 (Dense)                (None, 16)           9179152     flatten_5[0][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "dense_12 (Dense)                (None, 2)            34          dense_11[0][0]                   \n",
            "==================================================================================================\n",
            "Total params: 9,327,714\n",
            "Trainable params: 9,327,714\n",
            "Non-trainable params: 0\n",
            "__________________________________________________________________________________________________\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "id": "YNtu65vHWU2G",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "# without dilated convolution -> 4,526,978 parameters\n",
        "# with dilated convolution -> 3,875,714 (3)\n",
        "# 1,639,298 (dilation_rate = 11)\n",
        "# 2,683,714 (7)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "hy91iWJiedym",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "from keras.optimizers import Adam\n",
        "optim = Adam(lr=0.01)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "nU_BGGtyzLFY",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "model.compile(optimizer='rmsprop', loss = 'binary_crossentropy', metrics=['acc'])"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "yGZ0a73usyZ_",
        "colab_type": "code",
        "outputId": "6ced0295-22c6-425b-f85d-dfa0b151e120",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        }
      },
      "cell_type": "code",
      "source": [
        "np.mean(x2_tr)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "0.52803403"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 42
        }
      ]
    },
    {
      "metadata": {
        "id": "8vL1cWN20VWX",
        "colab_type": "code",
        "outputId": "960ed2ec-f4d0-48a4-fa66-7e9e513e4ecd",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1147
        }
      },
      "cell_type": "code",
      "source": [
        "hist = model.fit([x1_tr,x2_tr,x3_tr], y_tr, epochs = 30, batch_size = 32, validation_split=0.1)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Train on 820 samples, validate on 92 samples\n",
            "Epoch 1/30\n",
            "820/820 [==============================] - 50s 61ms/step - loss: 2.3373 - acc: 0.5134 - val_loss: 0.6881 - val_acc: 0.5435\n",
            "Epoch 2/30\n",
            "820/820 [==============================] - 36s 44ms/step - loss: 0.7039 - acc: 0.5195 - val_loss: 0.6720 - val_acc: 0.5543\n",
            "Epoch 3/30\n",
            "820/820 [==============================] - 36s 44ms/step - loss: 0.6917 - acc: 0.6390 - val_loss: 0.7427 - val_acc: 0.5543\n",
            "Epoch 4/30\n",
            "820/820 [==============================] - 36s 44ms/step - loss: 0.6282 - acc: 0.6976 - val_loss: 0.6037 - val_acc: 0.7283\n",
            "Epoch 5/30\n",
            "820/820 [==============================] - 37s 45ms/step - loss: 0.5652 - acc: 0.7829 - val_loss: 0.5289 - val_acc: 0.7609\n",
            "Epoch 6/30\n",
            "820/820 [==============================] - 37s 45ms/step - loss: 0.2836 - acc: 0.8756 - val_loss: 0.4702 - val_acc: 0.8478\n",
            "Epoch 7/30\n",
            "820/820 [==============================] - 36s 44ms/step - loss: 0.1876 - acc: 0.9183 - val_loss: 0.6607 - val_acc: 0.7717\n",
            "Epoch 8/30\n",
            "820/820 [==============================] - 36s 44ms/step - loss: 0.1328 - acc: 0.9512 - val_loss: 0.3350 - val_acc: 0.8804\n",
            "Epoch 9/30\n",
            "820/820 [==============================] - 36s 44ms/step - loss: 0.4081 - acc: 0.9146 - val_loss: 0.3433 - val_acc: 0.8587\n",
            "Epoch 10/30\n",
            "820/820 [==============================] - 36s 44ms/step - loss: 0.0694 - acc: 0.9780 - val_loss: 0.3437 - val_acc: 0.9130\n",
            "Epoch 11/30\n",
            "820/820 [==============================] - 36s 44ms/step - loss: 0.0270 - acc: 0.9902 - val_loss: 0.4387 - val_acc: 0.9130\n",
            "Epoch 12/30\n",
            "820/820 [==============================] - 36s 44ms/step - loss: 0.8010 - acc: 0.9366 - val_loss: 0.5970 - val_acc: 0.8478\n",
            "Epoch 13/30\n",
            "820/820 [==============================] - 36s 44ms/step - loss: 0.0136 - acc: 0.9963 - val_loss: 0.5644 - val_acc: 0.9022\n",
            "Epoch 14/30\n",
            "820/820 [==============================] - 36s 44ms/step - loss: 0.0507 - acc: 0.9866 - val_loss: 0.5209 - val_acc: 0.9022\n",
            "Epoch 15/30\n",
            "820/820 [==============================] - 36s 44ms/step - loss: 0.0223 - acc: 0.9915 - val_loss: 0.4685 - val_acc: 0.9239\n",
            "Epoch 16/30\n",
            "820/820 [==============================] - 36s 44ms/step - loss: 0.0026 - acc: 0.9988 - val_loss: 0.5634 - val_acc: 0.9239\n",
            "Epoch 17/30\n",
            "820/820 [==============================] - 36s 44ms/step - loss: 0.0020 - acc: 0.9988 - val_loss: 0.6134 - val_acc: 0.9130\n",
            "Epoch 18/30\n",
            "820/820 [==============================] - 36s 44ms/step - loss: 0.2877 - acc: 0.9598 - val_loss: 0.6259 - val_acc: 0.9022\n",
            "Epoch 19/30\n",
            "820/820 [==============================] - 36s 44ms/step - loss: 0.0014 - acc: 0.9988 - val_loss: 0.6617 - val_acc: 0.9022\n",
            "Epoch 20/30\n",
            "820/820 [==============================] - 36s 44ms/step - loss: 0.0010 - acc: 0.9988 - val_loss: 0.6922 - val_acc: 0.9130\n",
            "Epoch 21/30\n",
            "820/820 [==============================] - 36s 44ms/step - loss: 1.2160 - acc: 0.9098 - val_loss: 0.7892 - val_acc: 0.8696\n",
            "Epoch 22/30\n",
            "820/820 [==============================] - 36s 44ms/step - loss: 0.0141 - acc: 0.9927 - val_loss: 0.4981 - val_acc: 0.9130\n",
            "Epoch 23/30\n",
            "820/820 [==============================] - 36s 44ms/step - loss: 0.0016 - acc: 0.9988 - val_loss: 0.6301 - val_acc: 0.9022\n",
            "Epoch 24/30\n",
            "820/820 [==============================] - 36s 44ms/step - loss: 9.9257e-04 - acc: 0.9988 - val_loss: 0.5921 - val_acc: 0.9239\n",
            "Epoch 25/30\n",
            "820/820 [==============================] - 36s 44ms/step - loss: 8.9897e-04 - acc: 0.9988 - val_loss: 0.6718 - val_acc: 0.9239\n",
            "Epoch 26/30\n",
            "820/820 [==============================] - 36s 44ms/step - loss: 8.6519e-04 - acc: 0.9988 - val_loss: 0.7416 - val_acc: 0.9239\n",
            "Epoch 27/30\n",
            "820/820 [==============================] - 36s 44ms/step - loss: 8.5614e-04 - acc: 0.9988 - val_loss: 0.8020 - val_acc: 0.9239\n",
            "Epoch 28/30\n",
            "820/820 [==============================] - 36s 44ms/step - loss: 0.3760 - acc: 0.9683 - val_loss: 0.5426 - val_acc: 0.8913\n",
            "Epoch 29/30\n",
            "820/820 [==============================] - 36s 44ms/step - loss: 0.0010 - acc: 1.0000 - val_loss: 0.4847 - val_acc: 0.9022\n",
            "Epoch 30/30\n",
            "820/820 [==============================] - 36s 44ms/step - loss: 1.5042e-04 - acc: 1.0000 - val_loss: 0.4865 - val_acc: 0.9130\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "id": "nbVUv354poB4",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "x1_ts = np.array(x1_ts, dtype = 'float32')\n",
        "x2_ts = np.array(x2_ts, dtype = 'float32')\n",
        "x3_ts = np.array(x3_ts, dtype = 'float32')\n",
        "\n",
        "x1_ts = x1_ts/255.\n",
        "x2_ts = x2_ts/255.\n",
        "x3_ts = x3_ts/255.\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "yrZHmFB00jVH",
        "colab_type": "code",
        "outputId": "aadd34e6-b310-4839-a9d2-d123f3289f20",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 53
        }
      },
      "cell_type": "code",
      "source": [
        "model.evaluate([x1_ts,x2_ts,x3_ts], y_ts, batch_size = 64)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "228/228 [==============================] - 9s 38ms/step\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[1.3127837903928339, 0.8684210505401879]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 62
        }
      ]
    },
    {
      "metadata": {
        "id": "HqgZ2kTggJAS",
        "colab_type": "code",
        "outputId": "2d50ba63-c695-44cc-8523-0023633dcdc1",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 909
        }
      },
      "cell_type": "code",
      "source": [
        "model.summary()"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "__________________________________________________________________________________________________\n",
            "Layer (type)                    Output Shape         Param #     Connected to                     \n",
            "==================================================================================================\n",
            "input_16 (InputLayer)           (None, 224, 224, 3)  0                                            \n",
            "__________________________________________________________________________________________________\n",
            "input_17 (InputLayer)           (None, 224, 224, 3)  0                                            \n",
            "__________________________________________________________________________________________________\n",
            "input_18 (InputLayer)           (None, 224, 224, 3)  0                                            \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_71 (Conv2D)              (None, 224, 224, 64) 1792        input_16[0][0]                   \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_73 (Conv2D)              (None, 224, 224, 64) 1792        input_17[0][0]                   \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_75 (Conv2D)              (None, 224, 224, 64) 1792        input_18[0][0]                   \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_72 (Conv2D)              (None, 224, 224, 64) 36928       conv2d_71[0][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_74 (Conv2D)              (None, 224, 224, 64) 36928       conv2d_73[0][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_76 (Conv2D)              (None, 224, 224, 64) 36928       conv2d_75[0][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "concatenate_6 (Concatenate)     (None, 672, 224, 64) 0           conv2d_72[0][0]                  \n",
            "                                                                 conv2d_74[0][0]                  \n",
            "                                                                 conv2d_76[0][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_77 (Conv2D)              (None, 670, 222, 32) 18464       concatenate_6[0][0]              \n",
            "__________________________________________________________________________________________________\n",
            "activation_9 (Activation)       (None, 670, 222, 32) 0           conv2d_77[0][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_78 (Conv2D)              (None, 668, 220, 16) 4624        activation_9[0][0]               \n",
            "__________________________________________________________________________________________________\n",
            "activation_10 (Activation)      (None, 668, 220, 16) 0           conv2d_78[0][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "max_pooling2d_14 (MaxPooling2D) (None, 334, 110, 16) 0           activation_10[0][0]              \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_80 (Conv2D)              (None, 332, 108, 64) 9280        max_pooling2d_14[0][0]           \n",
            "__________________________________________________________________________________________________\n",
            "max_pooling2d_16 (MaxPooling2D) (None, 166, 54, 64)  0           conv2d_80[0][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "flatten_5 (Flatten)             (None, 573696)       0           max_pooling2d_16[0][0]           \n",
            "__________________________________________________________________________________________________\n",
            "dense_11 (Dense)                (None, 16)           9179152     flatten_5[0][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "dense_12 (Dense)                (None, 2)            34          dense_11[0][0]                   \n",
            "==================================================================================================\n",
            "Total params: 9,327,714\n",
            "Trainable params: 9,327,714\n",
            "Non-trainable params: 0\n",
            "__________________________________________________________________________________________________\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "id": "eCrP_DYrrTCv",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "a = hist.history"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "-cS3vi_H8ULS",
        "colab_type": "code",
        "outputId": "81f96677-7978-456b-9d4e-1f5a3eb2464b",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 563
        }
      },
      "cell_type": "code",
      "source": [
        "a['acc']"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[0.5134146335648327,\n",
              " 0.5195121954127055,\n",
              " 0.6390243905346568,\n",
              " 0.697560975609756,\n",
              " 0.7829268298498014,\n",
              " 0.8756097555160522,\n",
              " 0.9182926823453206,\n",
              " 0.9512195119043676,\n",
              " 0.914634146050709,\n",
              " 0.9780487804878049,\n",
              " 0.9902439024390244,\n",
              " 0.9365853655629042,\n",
              " 0.9963414634146341,\n",
              " 0.9865853658536585,\n",
              " 0.9914634146341463,\n",
              " 0.998780487804878,\n",
              " 0.998780487804878,\n",
              " 0.9597560975609756,\n",
              " 0.998780487804878,\n",
              " 0.998780487804878,\n",
              " 0.9097560975609756,\n",
              " 0.9926829268292683,\n",
              " 0.998780487804878,\n",
              " 0.998780487804878,\n",
              " 0.998780487804878,\n",
              " 0.998780487804878,\n",
              " 0.998780487804878,\n",
              " 0.9682926829268292,\n",
              " 1.0,\n",
              " 1.0]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 67
        }
      ]
    },
    {
      "metadata": {
        "id": "moRJ9q_-8VVJ",
        "colab_type": "code",
        "outputId": "2bdc5cdc-43bd-40ae-f670-721ef07fc0b9",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 563
        }
      },
      "cell_type": "code",
      "source": [
        "a['val_acc']"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[0.5434782660525778,\n",
              " 0.5543478312699691,\n",
              " 0.554347822199697,\n",
              " 0.7282608695652174,\n",
              " 0.7608695600343787,\n",
              " 0.8478260947310406,\n",
              " 0.7717391304347826,\n",
              " 0.8804347903832145,\n",
              " 0.8586956599484319,\n",
              " 0.9130434756693633,\n",
              " 0.9130434756693633,\n",
              " 0.8478260817735092,\n",
              " 0.9021739052689594,\n",
              " 0.9021739156349845,\n",
              " 0.923913035703742,\n",
              " 0.923913035703742,\n",
              " 0.9130434756693633,\n",
              " 0.9021739104519719,\n",
              " 0.9021739104519719,\n",
              " 0.9130434756693633,\n",
              " 0.8695652251658232,\n",
              " 0.9130434808523759,\n",
              " 0.9021739156349845,\n",
              " 0.9239130408867545,\n",
              " 0.9239130408867545,\n",
              " 0.9239130408867545,\n",
              " 0.9239130408867545,\n",
              " 0.8913043452345807,\n",
              " 0.9021739104519719,\n",
              " 0.9130434756693633]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 68
        }
      ]
    },
    {
      "metadata": {
        "id": "JAO3SH3s8tuZ",
        "colab_type": "code",
        "outputId": "29e33699-f5a8-470c-9c10-04fb3c8f571e",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 563
        }
      },
      "cell_type": "code",
      "source": [
        "a['loss']"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[2.337302986586966,\n",
              " 0.7039013632913915,\n",
              " 0.6916865659923088,\n",
              " 0.6282462563456559,\n",
              " 0.565231039902059,\n",
              " 0.28355068339080347,\n",
              " 0.1876298477010029,\n",
              " 0.13282960704187066,\n",
              " 0.4080822079646878,\n",
              " 0.06944416107200994,\n",
              " 0.02703175543571067,\n",
              " 0.8009825841455561,\n",
              " 0.013638574250678463,\n",
              " 0.05067387634615709,\n",
              " 0.022332115128363778,\n",
              " 0.0026225854509237517,\n",
              " 0.001998805815780609,\n",
              " 0.2876922585864455,\n",
              " 0.001359606504145842,\n",
              " 0.0010014969849369041,\n",
              " 1.2160179390048431,\n",
              " 0.014097257445299452,\n",
              " 0.0015622871340161598,\n",
              " 0.0009925666307769529,\n",
              " 0.0008989734537935208,\n",
              " 0.0008651933063627594,\n",
              " 0.0008561365404399728,\n",
              " 0.3760461639124792,\n",
              " 0.0010388923873302623,\n",
              " 0.00015041818850125132]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 69
        }
      ]
    },
    {
      "metadata": {
        "id": "mFeTt4cz8u-7",
        "colab_type": "code",
        "outputId": "5598ed2b-ed90-4d69-8fc2-dca41e13541c",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 563
        }
      },
      "cell_type": "code",
      "source": [
        "a['val_loss']"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[0.688093944736149,\n",
              " 0.6719964302104452,\n",
              " 0.7427116839782052,\n",
              " 0.603661386863045,\n",
              " 0.5288617766421774,\n",
              " 0.47019148909527325,\n",
              " 0.6606988855030226,\n",
              " 0.3349693054738252,\n",
              " 0.34327274820078973,\n",
              " 0.34370358353075775,\n",
              " 0.4386910822080529,\n",
              " 0.5970283383908479,\n",
              " 0.5644073952799258,\n",
              " 0.5208864471186763,\n",
              " 0.46848033562950464,\n",
              " 0.5633622848469279,\n",
              " 0.6134059766064519,\n",
              " 0.6259374774020651,\n",
              " 0.6617435553799504,\n",
              " 0.6922343787939652,\n",
              " 0.7891541304795638,\n",
              " 0.49805908099464746,\n",
              " 0.6301154235134954,\n",
              " 0.5920584435048311,\n",
              " 0.671793009923852,\n",
              " 0.7415646677431853,\n",
              " 0.8020420281783395,\n",
              " 0.5425950988479282,\n",
              " 0.4846626701562301,\n",
              " 0.48649566069893213]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 70
        }
      ]
    },
    {
      "metadata": {
        "id": "r0aaUvFs8whQ",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        ""
      ],
      "execution_count": 0,
      "outputs": []
    }
  ]
}