ventilator_gb_rescaling_eda.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "kernelspec": {
      "language": "python",
      "display_name": "Python 3",
      "name": "python3"
    },
    "language_info": {
      "name": "python",
      "version": "3.7.10",
      "mimetype": "text/x-python",
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "pygments_lexer": "ipython3",
      "nbconvert_exporter": "python",
      "file_extension": ".py"
    },
    "colab": {
      "name": "ventilator-gb-rescaling-eda.ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "machine_shape": "hm",
      "include_colab_link": true
    },
    "accelerator": "TPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/bobbercheng/ventilator-pressure-prediction/blob/master/ventilator_gb_rescaling_eda.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 354
        },
        "id": "uslq2jL49nGk",
        "outputId": "fa850d0c-64a8-4494-c0c8-21010f884f0c"
      },
      "source": [
        "# Update pandas version for Colab TPU\n",
        "#!pip install pandas==1.3.2"
      ],
      "execution_count": 1,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Collecting pandas==1.3.2\n",
            "  Downloading pandas-1.3.2-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (11.3 MB)\n",
            "\u001b[K     |████████████████████████████████| 11.3 MB 5.2 MB/s \n",
            "\u001b[?25hRequirement already satisfied: pytz>=2017.3 in /usr/local/lib/python3.7/dist-packages (from pandas==1.3.2) (2018.9)\n",
            "Requirement already satisfied: python-dateutil>=2.7.3 in /usr/local/lib/python3.7/dist-packages (from pandas==1.3.2) (2.8.2)\n",
            "Requirement already satisfied: numpy>=1.17.3 in /usr/local/lib/python3.7/dist-packages (from pandas==1.3.2) (1.19.5)\n",
            "Requirement already satisfied: six>=1.5 in /usr/local/lib/python3.7/dist-packages (from python-dateutil>=2.7.3->pandas==1.3.2) (1.15.0)\n",
            "Installing collected packages: pandas\n",
            "  Attempting uninstall: pandas\n",
            "    Found existing installation: pandas 1.1.5\n",
            "    Uninstalling pandas-1.1.5:\n",
            "      Successfully uninstalled pandas-1.1.5\n",
            "\u001b[31mERROR: pip's dependency resolver does not currently take into account all the packages that are installed. This behaviour is the source of the following dependency conflicts.\n",
            "google-colab 1.0.0 requires pandas~=1.1.0; python_version >= \"3.0\", but you have pandas 1.3.2 which is incompatible.\u001b[0m\n",
            "Successfully installed pandas-1.3.2\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.colab-display-data+json": {
              "pip_warning": {
                "packages": [
                  "pandas"
                ]
              }
            }
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "RBJlpUNI9com"
      },
      "source": [
        "## Import libraries"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "iXzLriPA9coq"
      },
      "source": [
        "try:\n",
        "  import google.colab\n",
        "  IN_COLAB = True\n",
        "except:\n",
        "  IN_COLAB = False"
      ],
      "execution_count": 1,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "9esF__aG9cor",
        "outputId": "9bbe7ff2-f4bb-4aac-f8de-49cb6bbf3144"
      },
      "source": [
        "if IN_COLAB:\n",
        "  from google.colab import drive\n",
        "  drive.mount(\"/gdrive\", force_remount=True)"
      ],
      "execution_count": 2,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Mounted at /gdrive\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "MZ_f3Z2M9cos",
        "outputId": "3dadabb2-0b8e-4bca-cdce-a25049d28da5"
      },
      "source": [
        "if IN_COLAB:\n",
        "  !pip install kaggle\n",
        "  !mkdir /root/.kaggle\n",
        "  !cp /gdrive/MyDrive/ventilator-pressure-prediction/kaggle.json /root/.kaggle\n",
        "  !kaggle competitions download -c ventilator-pressure-prediction\n",
        "  !mkdir -p ../input/ventilator-pressure-prediction\n",
        "  !unzip '*.zip' -d ../input/ventilator-pressure-prediction\n",
        "  !ls ../input/ventilator-pressure-prediction"
      ],
      "execution_count": 3,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Requirement already satisfied: kaggle in /usr/local/lib/python3.7/dist-packages (1.5.12)\n",
            "Requirement already satisfied: six>=1.10 in /usr/local/lib/python3.7/dist-packages (from kaggle) (1.15.0)\n",
            "Requirement already satisfied: requests in /usr/local/lib/python3.7/dist-packages (from kaggle) (2.23.0)\n",
            "Requirement already satisfied: certifi in /usr/local/lib/python3.7/dist-packages (from kaggle) (2021.5.30)\n",
            "Requirement already satisfied: urllib3 in /usr/local/lib/python3.7/dist-packages (from kaggle) (1.24.3)\n",
            "Requirement already satisfied: python-dateutil in /usr/local/lib/python3.7/dist-packages (from kaggle) (2.8.2)\n",
            "Requirement already satisfied: tqdm in /usr/local/lib/python3.7/dist-packages (from kaggle) (4.62.3)\n",
            "Requirement already satisfied: python-slugify in /usr/local/lib/python3.7/dist-packages (from kaggle) (5.0.2)\n",
            "Requirement already satisfied: text-unidecode>=1.3 in /usr/local/lib/python3.7/dist-packages (from python-slugify->kaggle) (1.3)\n",
            "Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.7/dist-packages (from requests->kaggle) (3.0.4)\n",
            "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.7/dist-packages (from requests->kaggle) (2.10)\n",
            "Warning: Looks like you're using an outdated API Version, please consider updating (server 1.5.12 / client 1.5.4)\n",
            "Downloading sample_submission.csv.zip to /content\n",
            "  0% 0.00/8.50M [00:00<?, ?B/s]\n",
            "100% 8.50M/8.50M [00:00<00:00, 141MB/s]\n",
            "Downloading train.csv.zip to /content\n",
            " 96% 133M/139M [00:00<00:00, 223MB/s]\n",
            "100% 139M/139M [00:00<00:00, 219MB/s]\n",
            "Downloading test.csv.zip to /content\n",
            " 76% 57.0M/75.4M [00:00<00:00, 164MB/s]\n",
            "100% 75.4M/75.4M [00:00<00:00, 190MB/s]\n",
            "Archive:  train.csv.zip\n",
            "  inflating: ../input/ventilator-pressure-prediction/train.csv  \n",
            "\n",
            "Archive:  sample_submission.csv.zip\n",
            "  inflating: ../input/ventilator-pressure-prediction/sample_submission.csv  \n",
            "\n",
            "Archive:  test.csv.zip\n",
            "  inflating: ../input/ventilator-pressure-prediction/test.csv  \n",
            "\n",
            "3 archives were successfully processed.\n",
            "sample_submission.csv  test.csv  train.csv\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "execution": {
          "iopub.status.busy": "2021-10-21T17:30:55.782617Z",
          "iopub.execute_input": "2021-10-21T17:30:55.783998Z",
          "iopub.status.idle": "2021-10-21T17:30:55.793669Z",
          "shell.execute_reply.started": "2021-10-21T17:30:55.783937Z",
          "shell.execute_reply": "2021-10-21T17:30:55.792667Z"
        },
        "trusted": true,
        "id": "1TfjZidC9cos"
      },
      "source": [
        "import gc\n",
        "import numpy as np\n",
        "import pandas as pd\n",
        "import random\n",
        "import os\n",
        "from sklearn.model_selection import KFold\n",
        "from sklearn.preprocessing import RobustScaler\n",
        "from sklearn.metrics import mean_absolute_error\n",
        "\n",
        "import tensorflow as tf\n",
        "from tensorflow.keras.utils import plot_model\n",
        "from tensorflow.keras.models import Model, load_model\n",
        "from tensorflow.keras.callbacks import EarlyStopping\n",
        "from tensorflow.keras.callbacks import ModelCheckpoint\n",
        "from tensorflow.keras.callbacks import ReduceLROnPlateau\n",
        "from tensorflow.keras.layers import Bidirectional, LSTM\n",
        "from tensorflow.keras.layers import Dense, Dropout, Input\n",
        "from tensorflow.keras.layers import Concatenate, Add, GRU\n",
        "\n",
        "# Random Seed Initialize\n",
        "RANDOM_SEED = 42\n",
        "\n",
        "def seed_everything(seed=RANDOM_SEED):\n",
        "    os.environ['PYTHONHASHSEED'] = str(seed)\n",
        "    np.random.seed(seed)\n",
        "    random.seed(seed)\n",
        "    tf.random.set_seed(seed)\n",
        "    \n",
        "seed_everything()"
      ],
      "execution_count": 5,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "XycyrO7g9cot"
      },
      "source": [
        "## Load datasets"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "execution": {
          "iopub.status.busy": "2021-10-21T17:24:55.352925Z",
          "iopub.execute_input": "2021-10-21T17:24:55.353455Z",
          "iopub.status.idle": "2021-10-21T17:25:06.101910Z",
          "shell.execute_reply.started": "2021-10-21T17:24:55.353418Z",
          "shell.execute_reply": "2021-10-21T17:25:06.100969Z"
        },
        "trusted": true,
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 220
        },
        "id": "p_vUMe_39cot",
        "outputId": "338c482f-2e94-47a2-f725-c85a5870f023"
      },
      "source": [
        "train_df = pd.read_csv('../input/ventilator-pressure-prediction/train.csv')\n",
        "print(f\"train_df: {train_df.shape}\")\n",
        "train_df.head()"
      ],
      "execution_count": 9,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "train_df: (6036000, 8)\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>id</th>\n",
              "      <th>breath_id</th>\n",
              "      <th>R</th>\n",
              "      <th>C</th>\n",
              "      <th>time_step</th>\n",
              "      <th>u_in</th>\n",
              "      <th>u_out</th>\n",
              "      <th>pressure</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>20</td>\n",
              "      <td>50</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.083334</td>\n",
              "      <td>0</td>\n",
              "      <td>5.837492</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>2</td>\n",
              "      <td>1</td>\n",
              "      <td>20</td>\n",
              "      <td>50</td>\n",
              "      <td>0.033652</td>\n",
              "      <td>18.383041</td>\n",
              "      <td>0</td>\n",
              "      <td>5.907794</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>3</td>\n",
              "      <td>1</td>\n",
              "      <td>20</td>\n",
              "      <td>50</td>\n",
              "      <td>0.067514</td>\n",
              "      <td>22.509278</td>\n",
              "      <td>0</td>\n",
              "      <td>7.876254</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>4</td>\n",
              "      <td>1</td>\n",
              "      <td>20</td>\n",
              "      <td>50</td>\n",
              "      <td>0.101542</td>\n",
              "      <td>22.808822</td>\n",
              "      <td>0</td>\n",
              "      <td>11.742872</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>5</td>\n",
              "      <td>1</td>\n",
              "      <td>20</td>\n",
              "      <td>50</td>\n",
              "      <td>0.135756</td>\n",
              "      <td>25.355850</td>\n",
              "      <td>0</td>\n",
              "      <td>12.234987</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "   id  breath_id   R   C  time_step       u_in  u_out   pressure\n",
              "0   1          1  20  50   0.000000   0.083334      0   5.837492\n",
              "1   2          1  20  50   0.033652  18.383041      0   5.907794\n",
              "2   3          1  20  50   0.067514  22.509278      0   7.876254\n",
              "3   4          1  20  50   0.101542  22.808822      0  11.742872\n",
              "4   5          1  20  50   0.135756  25.355850      0  12.234987"
            ]
          },
          "metadata": {},
          "execution_count": 9
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "execution": {
          "iopub.status.busy": "2021-10-21T17:25:11.213742Z",
          "iopub.execute_input": "2021-10-21T17:25:11.214675Z",
          "iopub.status.idle": "2021-10-21T17:25:11.297152Z",
          "shell.execute_reply.started": "2021-10-21T17:25:11.214617Z",
          "shell.execute_reply": "2021-10-21T17:25:11.295985Z"
        },
        "trusted": true,
        "id": "R5oBMjYJ9cov"
      },
      "source": [
        "all_pressure = np.sort(train_df['pressure'].unique())\n",
        "pressure_min =  all_pressure[0].item()\n",
        "pressure_max = all_pressure[-1].item()\n",
        "pressure_step = (all_pressure[1] - all_pressure[0]).item()"
      ],
      "execution_count": 10,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "80hIcGg09cou"
      },
      "source": [
        "DEBUG = True\n",
        "\n",
        "if DEBUG:\n",
        "    train_df = train_df[:80*10000]"
      ],
      "execution_count": 11,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "execution": {
          "iopub.status.busy": "2021-10-21T17:25:06.103441Z",
          "iopub.execute_input": "2021-10-21T17:25:06.104426Z",
          "iopub.status.idle": "2021-10-21T17:25:11.211412Z",
          "shell.execute_reply.started": "2021-10-21T17:25:06.104376Z",
          "shell.execute_reply": "2021-10-21T17:25:11.209506Z"
        },
        "trusted": true,
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 220
        },
        "id": "y-0XhQ3a9cov",
        "outputId": "f126fd21-48a3-424c-fbd7-7d0afb99a891"
      },
      "source": [
        "test_df = pd.read_csv('../input/ventilator-pressure-prediction/test.csv')\n",
        "print(f\"test_df: {test_df.shape}\")\n",
        "test_df.head()"
      ],
      "execution_count": 12,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "test_df: (4024000, 7)\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>id</th>\n",
              "      <th>breath_id</th>\n",
              "      <th>R</th>\n",
              "      <th>C</th>\n",
              "      <th>time_step</th>\n",
              "      <th>u_in</th>\n",
              "      <th>u_out</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>5</td>\n",
              "      <td>20</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>2</td>\n",
              "      <td>0</td>\n",
              "      <td>5</td>\n",
              "      <td>20</td>\n",
              "      <td>0.031904</td>\n",
              "      <td>7.515046</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>3</td>\n",
              "      <td>0</td>\n",
              "      <td>5</td>\n",
              "      <td>20</td>\n",
              "      <td>0.063827</td>\n",
              "      <td>14.651675</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>4</td>\n",
              "      <td>0</td>\n",
              "      <td>5</td>\n",
              "      <td>20</td>\n",
              "      <td>0.095751</td>\n",
              "      <td>21.230610</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>5</td>\n",
              "      <td>0</td>\n",
              "      <td>5</td>\n",
              "      <td>20</td>\n",
              "      <td>0.127644</td>\n",
              "      <td>26.320956</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "   id  breath_id  R   C  time_step       u_in  u_out\n",
              "0   1          0  5  20   0.000000   0.000000      0\n",
              "1   2          0  5  20   0.031904   7.515046      0\n",
              "2   3          0  5  20   0.063827  14.651675      0\n",
              "3   4          0  5  20   0.095751  21.230610      0\n",
              "4   5          0  5  20   0.127644  26.320956      0"
            ]
          },
          "metadata": {},
          "execution_count": 12
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "IVPWgQDL9cow"
      },
      "source": [
        "## Feature Engineering"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "execution": {
          "iopub.status.busy": "2021-10-21T17:25:11.298767Z",
          "iopub.execute_input": "2021-10-21T17:25:11.299164Z",
          "iopub.status.idle": "2021-10-21T17:27:48.006762Z",
          "shell.execute_reply.started": "2021-10-21T17:25:11.299125Z",
          "shell.execute_reply": "2021-10-21T17:27:48.005658Z"
        },
        "trusted": true,
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "1NHNQSZp9cow",
        "outputId": "8a7d9e47-bef1-4e85-b101-ccbdef1ef3fa"
      },
      "source": [
        "def add_features(df):\n",
        "    df['cross']= df['u_in'] * df['u_out']\n",
        "    df['cross2']= df['time_step'] * df['u_out']\n",
        "    df['area'] = df['time_step'] * df['u_in']\n",
        "    df['area'] = df.groupby('breath_id')['area'].cumsum()\n",
        "    df['time_step_cumsum'] = df.groupby(['breath_id'])['time_step'].cumsum()\n",
        "    df['u_in_cumsum'] = (df['u_in']).groupby(df['breath_id']).cumsum()\n",
        "    print(\"Step-1...Completed\")\n",
        "    \n",
        "    df['u_in_lag1'] = df.groupby('breath_id')['u_in'].shift(1)\n",
        "    df['u_out_lag1'] = df.groupby('breath_id')['u_out'].shift(1)\n",
        "    df['u_in_lag_back1'] = df.groupby('breath_id')['u_in'].shift(-1)\n",
        "    df['u_out_lag_back1'] = df.groupby('breath_id')['u_out'].shift(-1)\n",
        "    df['u_in_lag2'] = df.groupby('breath_id')['u_in'].shift(2)\n",
        "    df['u_out_lag2'] = df.groupby('breath_id')['u_out'].shift(2)\n",
        "    df['u_in_lag_back2'] = df.groupby('breath_id')['u_in'].shift(-2)\n",
        "    df['u_out_lag_back2'] = df.groupby('breath_id')['u_out'].shift(-2)\n",
        "    df['u_in_lag3'] = df.groupby('breath_id')['u_in'].shift(3)\n",
        "    df['u_out_lag3'] = df.groupby('breath_id')['u_out'].shift(3)\n",
        "    df['u_in_lag_back3'] = df.groupby('breath_id')['u_in'].shift(-3)\n",
        "    df['u_out_lag_back3'] = df.groupby('breath_id')['u_out'].shift(-3)\n",
        "    df['u_in_lag4'] = df.groupby('breath_id')['u_in'].shift(4)\n",
        "    df['u_out_lag4'] = df.groupby('breath_id')['u_out'].shift(4)\n",
        "    df['u_in_lag_back4'] = df.groupby('breath_id')['u_in'].shift(-4)\n",
        "    df['u_out_lag_back4'] = df.groupby('breath_id')['u_out'].shift(-4)\n",
        "    df = df.fillna(0)\n",
        "    print(\"Step-2...Completed\")\n",
        "    \n",
        "    df['breath_id__u_in__max'] = df.groupby(['breath_id'])['u_in'].transform('max')\n",
        "    df['breath_id__u_out__max'] = df.groupby(['breath_id'])['u_out'].transform('max')\n",
        "    df['breath_id__u_in__diffmax'] = df.groupby(['breath_id'])['u_in'].transform('max') - df['u_in']\n",
        "    df['breath_id__u_in__diffmean'] = df.groupby(['breath_id'])['u_in'].transform('mean') - df['u_in']\n",
        "    df['breath_id__u_in__diffmax'] = df.groupby(['breath_id'])['u_in'].transform('max') - df['u_in']\n",
        "    df['breath_id__u_in__diffmean'] = df.groupby(['breath_id'])['u_in'].transform('mean') - df['u_in']\n",
        "    print(\"Step-3...Completed\")\n",
        "    \n",
        "    df['u_in_diff1'] = df['u_in'] - df['u_in_lag1']\n",
        "    df['u_out_diff1'] = df['u_out'] - df['u_out_lag1']\n",
        "    df['u_in_diff2'] = df['u_in'] - df['u_in_lag2']\n",
        "    df['u_out_diff2'] = df['u_out'] - df['u_out_lag2']\n",
        "    df['u_in_diff3'] = df['u_in'] - df['u_in_lag3']\n",
        "    df['u_out_diff3'] = df['u_out'] - df['u_out_lag3']\n",
        "    df['u_in_diff4'] = df['u_in'] - df['u_in_lag4']\n",
        "    df['u_out_diff4'] = df['u_out'] - df['u_out_lag4']\n",
        "    print(\"Step-4...Completed\")\n",
        "    \n",
        "    df['one'] = 1\n",
        "    df['count'] = (df['one']).groupby(df['breath_id']).cumsum()\n",
        "    df['u_in_cummean'] =df['u_in_cumsum'] /df['count']\n",
        "    \n",
        "    df['breath_id_lag']=df['breath_id'].shift(1).fillna(0)\n",
        "    df['breath_id_lag2']=df['breath_id'].shift(2).fillna(0)\n",
        "    df['breath_id_lagsame']=np.select([df['breath_id_lag']==df['breath_id']],[1],0)\n",
        "    df['breath_id_lag2same']=np.select([df['breath_id_lag2']==df['breath_id']],[1],0)\n",
        "    df['breath_id__u_in_lag'] = df['u_in'].shift(1).fillna(0)\n",
        "    df['breath_id__u_in_lag'] = df['breath_id__u_in_lag'] * df['breath_id_lagsame']\n",
        "    df['breath_id__u_in_lag2'] = df['u_in'].shift(2).fillna(0)\n",
        "    df['breath_id__u_in_lag2'] = df['breath_id__u_in_lag2'] * df['breath_id_lag2same']\n",
        "    print(\"Step-5...Completed\")\n",
        "    \n",
        "    df['time_step_diff'] = df.groupby('breath_id')['time_step'].diff().fillna(0)\n",
        "    df['ewm_u_in_mean'] = (df\\\n",
        "                           .groupby('breath_id')['u_in']\\\n",
        "                           .ewm(halflife=9)\\\n",
        "                           .mean()\\\n",
        "                           .reset_index(level=0,drop=True))\n",
        "    df[[\"15_in_sum\",\"15_in_min\",\"15_in_max\",\"15_in_mean\"]] = (df\\\n",
        "                                                              .groupby('breath_id')['u_in']\\\n",
        "                                                              .rolling(window=15,min_periods=1)\\\n",
        "                                                              .agg({\"15_in_sum\":\"sum\",\n",
        "                                                                    \"15_in_min\":\"min\",\n",
        "                                                                    \"15_in_max\":\"max\",\n",
        "                                                                    \"15_in_mean\":\"mean\"\n",
        "                                                                    #\"15_in_std\":\"std\"\n",
        "                                                               })\\\n",
        "                                                               .reset_index(level=0,drop=True))\n",
        "    print(\"Step-6...Completed\")\n",
        "    \n",
        "    #df['u_in_diff_1_2'] = df['u_in_lag1'] - df['u_in_lag2']\n",
        "    #df['u_out_diff_1_2'] = df['u_out_lag1'] - df['u_out_lag2']\n",
        "    #df['u_in_lagback_diff_1_2'] = df['u_in_lag_back1'] - df['u_in_lag_back2']\n",
        "    #df['u_out_lagback_diff_1_2'] = df['u_out_lag_back1'] - df['u_out_lag_back2']\n",
        "    \n",
        "    df['u_in_lagback_diff1'] = df['u_in'] - df['u_in_lag_back1']\n",
        "    df['u_out_lagback_diff1'] = df['u_out'] - df['u_out_lag_back1']\n",
        "    df['u_in_lagback_diff2'] = df['u_in'] - df['u_in_lag_back2']\n",
        "    df['u_out_lagback_diff2'] = df['u_out'] - df['u_out_lag_back2']\n",
        "    print(\"Step-7...Completed\")\n",
        "    \n",
        "    df['R'] = df['R'].astype(str)\n",
        "    df['C'] = df['C'].astype(str)\n",
        "#     df['R_cat'] = df['R'].astype(str)\n",
        "#     df['C_cat'] = df['C'].astype(str)\n",
        "    df['R__C'] = df[\"R\"].astype(str) + '__' + df[\"C\"].astype(str)\n",
        "    df = pd.get_dummies(df)\n",
        "    print(\"Step-8...Completed\")\n",
        "    \n",
        "    return df\n",
        "\n",
        "\n",
        "print(\"Train data...\\n\")\n",
        "train = add_features(train_df)\n",
        "\n",
        "print(\"\\nTest data...\\n\")\n",
        "test = add_features(test_df)"
      ],
      "execution_count": 13,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Train data...\n",
            "\n",
            "Step-1...Completed\n",
            "Step-2...Completed\n",
            "Step-3...Completed\n",
            "Step-4...Completed\n",
            "Step-5...Completed\n",
            "Step-6...Completed\n",
            "Step-7...Completed\n",
            "Step-8...Completed\n",
            "\n",
            "Test data...\n",
            "\n",
            "Step-1...Completed\n",
            "Step-2...Completed\n",
            "Step-3...Completed\n",
            "Step-4...Completed\n",
            "Step-5...Completed\n",
            "Step-6...Completed\n",
            "Step-7...Completed\n",
            "Step-8...Completed\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "execution": {
          "iopub.status.busy": "2021-10-21T17:28:07.762733Z",
          "iopub.execute_input": "2021-10-21T17:28:07.763099Z",
          "iopub.status.idle": "2021-10-21T17:28:07.795291Z",
          "shell.execute_reply.started": "2021-10-21T17:28:07.763065Z",
          "shell.execute_reply": "2021-10-21T17:28:07.794126Z"
        },
        "trusted": true,
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 223
        },
        "id": "PgRvxlmB9co0",
        "outputId": "cf0e1cc6-c36f-40ab-ff02-9d7823f2455f"
      },
      "source": [
        "train_df.head()"
      ],
      "execution_count": 14,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>id</th>\n",
              "      <th>breath_id</th>\n",
              "      <th>R</th>\n",
              "      <th>C</th>\n",
              "      <th>time_step</th>\n",
              "      <th>u_in</th>\n",
              "      <th>u_out</th>\n",
              "      <th>pressure</th>\n",
              "      <th>cross</th>\n",
              "      <th>cross2</th>\n",
              "      <th>area</th>\n",
              "      <th>time_step_cumsum</th>\n",
              "      <th>u_in_cumsum</th>\n",
              "      <th>u_in_lag1</th>\n",
              "      <th>u_out_lag1</th>\n",
              "      <th>u_in_lag_back1</th>\n",
              "      <th>u_out_lag_back1</th>\n",
              "      <th>u_in_lag2</th>\n",
              "      <th>u_out_lag2</th>\n",
              "      <th>u_in_lag_back2</th>\n",
              "      <th>u_out_lag_back2</th>\n",
              "      <th>u_in_lag3</th>\n",
              "      <th>u_out_lag3</th>\n",
              "      <th>u_in_lag_back3</th>\n",
              "      <th>u_out_lag_back3</th>\n",
              "      <th>u_in_lag4</th>\n",
              "      <th>u_out_lag4</th>\n",
              "      <th>u_in_lag_back4</th>\n",
              "      <th>u_out_lag_back4</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>20</td>\n",
              "      <td>50</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.083334</td>\n",
              "      <td>0</td>\n",
              "      <td>5.837492</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.083334</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>18.383041</td>\n",
              "      <td>0.0</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>22.509278</td>\n",
              "      <td>0.0</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>22.808822</td>\n",
              "      <td>0.0</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>25.355850</td>\n",
              "      <td>0.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>2</td>\n",
              "      <td>1</td>\n",
              "      <td>20</td>\n",
              "      <td>50</td>\n",
              "      <td>0.033652</td>\n",
              "      <td>18.383041</td>\n",
              "      <td>0</td>\n",
              "      <td>5.907794</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.618632</td>\n",
              "      <td>0.033652</td>\n",
              "      <td>18.466375</td>\n",
              "      <td>0.083334</td>\n",
              "      <td>0.0</td>\n",
              "      <td>22.509278</td>\n",
              "      <td>0.0</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>22.808822</td>\n",
              "      <td>0.0</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>25.355850</td>\n",
              "      <td>0.0</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>27.259866</td>\n",
              "      <td>0.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>3</td>\n",
              "      <td>1</td>\n",
              "      <td>20</td>\n",
              "      <td>50</td>\n",
              "      <td>0.067514</td>\n",
              "      <td>22.509278</td>\n",
              "      <td>0</td>\n",
              "      <td>7.876254</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>2.138333</td>\n",
              "      <td>0.101167</td>\n",
              "      <td>40.975653</td>\n",
              "      <td>18.383041</td>\n",
              "      <td>0.0</td>\n",
              "      <td>22.808822</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.083334</td>\n",
              "      <td>0.0</td>\n",
              "      <td>25.355850</td>\n",
              "      <td>0.0</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>27.259866</td>\n",
              "      <td>0.0</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>27.127486</td>\n",
              "      <td>0.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>4</td>\n",
              "      <td>1</td>\n",
              "      <td>20</td>\n",
              "      <td>50</td>\n",
              "      <td>0.101542</td>\n",
              "      <td>22.808822</td>\n",
              "      <td>0</td>\n",
              "      <td>11.742872</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>4.454391</td>\n",
              "      <td>0.202709</td>\n",
              "      <td>63.784476</td>\n",
              "      <td>22.509278</td>\n",
              "      <td>0.0</td>\n",
              "      <td>25.355850</td>\n",
              "      <td>0.0</td>\n",
              "      <td>18.383041</td>\n",
              "      <td>0.0</td>\n",
              "      <td>27.259866</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.083334</td>\n",
              "      <td>0.0</td>\n",
              "      <td>27.127486</td>\n",
              "      <td>0.0</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>26.807732</td>\n",
              "      <td>0.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>5</td>\n",
              "      <td>1</td>\n",
              "      <td>20</td>\n",
              "      <td>50</td>\n",
              "      <td>0.135756</td>\n",
              "      <td>25.355850</td>\n",
              "      <td>0</td>\n",
              "      <td>12.234987</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>7.896588</td>\n",
              "      <td>0.338464</td>\n",
              "      <td>89.140326</td>\n",
              "      <td>22.808822</td>\n",
              "      <td>0.0</td>\n",
              "      <td>27.259866</td>\n",
              "      <td>0.0</td>\n",
              "      <td>22.509278</td>\n",
              "      <td>0.0</td>\n",
              "      <td>27.127486</td>\n",
              "      <td>0.0</td>\n",
              "      <td>18.383041</td>\n",
              "      <td>0.0</td>\n",
              "      <td>26.807732</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.083334</td>\n",
              "      <td>0.0</td>\n",
              "      <td>27.864715</td>\n",
              "      <td>0.0</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "   id  breath_id   R  ...  u_out_lag4  u_in_lag_back4  u_out_lag_back4\n",
              "0   1          1  20  ...         NaN       25.355850              0.0\n",
              "1   2          1  20  ...         NaN       27.259866              0.0\n",
              "2   3          1  20  ...         NaN       27.127486              0.0\n",
              "3   4          1  20  ...         NaN       26.807732              0.0\n",
              "4   5          1  20  ...         0.0       27.864715              0.0\n",
              "\n",
              "[5 rows x 29 columns]"
            ]
          },
          "metadata": {},
          "execution_count": 14
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Zcyv-iig9co0",
        "outputId": "05235793-5f5a-4a51-86cd-5d58975eed12"
      },
      "source": [
        "del train_df\n",
        "del test_df\n",
        "gc.collect()"
      ],
      "execution_count": 15,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "245"
            ]
          },
          "metadata": {},
          "execution_count": 15
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "qip05b0r9co0",
        "outputId": "a0a3227a-d87c-46f4-d320-a2d4282e6635"
      },
      "source": [
        "targets = train[['pressure']].to_numpy().reshape(-1, 80)\n",
        "\n",
        "train.drop(['pressure','id', 'breath_id','one','count',\n",
        "            'breath_id_lag','breath_id_lag2','breath_id_lagsame',\n",
        "            'breath_id_lag2same'], axis=1, inplace=True)\n",
        "\n",
        "test = test.drop(['id', 'breath_id','one','count','breath_id_lag',\n",
        "                  'breath_id_lag2','breath_id_lagsame',\n",
        "                  'breath_id_lag2same'], axis=1)\n",
        "\n",
        "print(f\"train: {train.shape} \\ntest: {test.shape}\")"
      ],
      "execution_count": 16,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "train: (800000, 64) \n",
            "test: (4024000, 64)\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "OYWzQtL49co1",
        "outputId": "cc3a8963-8988-4ef4-dc79-ab17fa8d2102"
      },
      "source": [
        "scaler = RobustScaler()\n",
        "train = scaler.fit_transform(train)\n",
        "test = scaler.transform(test)\n",
        "\n",
        "train = train.reshape(-1, 80, train.shape[-1])\n",
        "test = test.reshape(-1, 80, train.shape[-1])\n",
        "\n",
        "print(f\"train: {train.shape} \\ntest: {test.shape} \\ntargets: {targets.shape}\")"
      ],
      "execution_count": 17,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "train: (10000, 80, 64) \n",
            "test: (50300, 80, 64) \n",
            "targets: (10000, 80)\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Osu14DeV9co1"
      },
      "source": [
        "## Hardware config"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "CisxazED9co1",
        "outputId": "fad0eefe-77dc-4c1e-9bf3-b773cae117e2"
      },
      "source": [
        "try:\n",
        "    tpu = tf.distribute.cluster_resolver.TPUClusterResolver()\n",
        "    tf.config.experimental_connect_to_cluster(tpu)\n",
        "    tf.tpu.experimental.initialize_tpu_system(tpu)\n",
        "    tpu_strategy = tf.distribute.experimental.TPUStrategy(tpu)\n",
        "    BATCH_SIZE = tpu_strategy.num_replicas_in_sync * 64\n",
        "    print(\"Running on TPU:\", tpu.master())\n",
        "    print(f\"Batch Size: {BATCH_SIZE}\")\n",
        "    \n",
        "except ValueError:\n",
        "    tpu_strategy = tf.distribute.get_strategy()\n",
        "    BATCH_SIZE = 512\n",
        "    print(f\"Running on {strategy.num_replicas_in_sync} replicas\")\n",
        "    print(f\"Batch Size: {BATCH_SIZE}\")"
      ],
      "execution_count": 18,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "INFO:tensorflow:Clearing out eager caches\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "INFO:tensorflow:Clearing out eager caches\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "INFO:tensorflow:Initializing the TPU system: grpc://10.21.194.242:8470\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "INFO:tensorflow:Initializing the TPU system: grpc://10.21.194.242:8470\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "INFO:tensorflow:Finished initializing TPU system.\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "INFO:tensorflow:Finished initializing TPU system.\n",
            "WARNING:absl:`tf.distribute.experimental.TPUStrategy` is deprecated, please use  the non experimental symbol `tf.distribute.TPUStrategy` instead.\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "INFO:tensorflow:Found TPU system:\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "INFO:tensorflow:Found TPU system:\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "INFO:tensorflow:*** Num TPU Cores: 8\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "INFO:tensorflow:*** Num TPU Cores: 8\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "INFO:tensorflow:*** Num TPU Workers: 1\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "INFO:tensorflow:*** Num TPU Workers: 1\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "INFO:tensorflow:*** Num TPU Cores Per Worker: 8\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "INFO:tensorflow:*** Num TPU Cores Per Worker: 8\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:localhost/replica:0/task:0/device:CPU:0, CPU, 0, 0)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:localhost/replica:0/task:0/device:CPU:0, CPU, 0, 0)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:CPU:0, CPU, 0, 0)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:CPU:0, CPU, 0, 0)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU:0, TPU, 0, 0)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU:0, TPU, 0, 0)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU:1, TPU, 0, 0)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU:1, TPU, 0, 0)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU:2, TPU, 0, 0)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU:2, TPU, 0, 0)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU:3, TPU, 0, 0)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU:3, TPU, 0, 0)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU:4, TPU, 0, 0)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU:4, TPU, 0, 0)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU:5, TPU, 0, 0)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU:5, TPU, 0, 0)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU:6, TPU, 0, 0)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU:6, TPU, 0, 0)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU:7, TPU, 0, 0)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU:7, TPU, 0, 0)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU_SYSTEM:0, TPU_SYSTEM, 0, 0)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU_SYSTEM:0, TPU_SYSTEM, 0, 0)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:XLA_CPU:0, XLA_CPU, 0, 0)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:XLA_CPU:0, XLA_CPU, 0, 0)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Running on TPU: grpc://10.21.194.242:8470\n",
            "Batch Size: 512\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "GAZMw10A9co1"
      },
      "source": [
        "## Rescaling layer"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "qLou5BDt9co2"
      },
      "source": [
        "@tf.custom_gradient\n",
        "def round_with_gradients(x):\n",
        "    def grad(dy):\n",
        "        return dy\n",
        "    return tf.round(x), grad\n",
        "\n",
        "class ScaleLayer(tf.keras.layers.Layer):\n",
        "    def __init__(self):\n",
        "        super(ScaleLayer, self).__init__()\n",
        "        self.min = tf.constant(pressure_min, dtype=np.float32)\n",
        "        self.max = tf.constant(pressure_max, dtype=np.float32)\n",
        "        self.step = tf.constant(pressure_step, dtype=np.float32)\n",
        "\n",
        "    def call(self, inputs):\n",
        "        steps = tf.math.divide(tf.math.add(inputs, -self.min), self.step)\n",
        "        int_steps = round_with_gradients(steps)\n",
        "        rescaled_steps = tf.math.add(tf.math.multiply(int_steps, self.step), self.min)\n",
        "        clipped = tf.clip_by_value(rescaled_steps, self.min, self.max)\n",
        "        return clipped"
      ],
      "execution_count": 19,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "5Ft0JBPC9co2"
      },
      "source": [
        "## Keras DNN Model"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "_HuMFGY39co2"
      },
      "source": [
        "def dnn_model():\n",
        "    \n",
        "    x_input = Input(shape=(train.shape[-2:]))\n",
        "    \n",
        "    x1 = Bidirectional(LSTM(units=768, return_sequences=True))(x_input)\n",
        "    x2 = Bidirectional(LSTM(units=512, return_sequences=True))(x1)\n",
        "    x3 = Bidirectional(LSTM(units=256, return_sequences=True))(x2)\n",
        "    x4 = Bidirectional(LSTM(units=128, return_sequences=True))(x3)\n",
        "    \n",
        "    z2 = Bidirectional(GRU(units=256, return_sequences=True))(x2)\n",
        "    z3 = Bidirectional(GRU(units=128, return_sequences=True))(Add()([x3, z2]))\n",
        "    z4 = Bidirectional(GRU(units=64, return_sequences=True))(Add()([x4, z3]))\n",
        "    \n",
        "    x = Concatenate(axis=2)([x4, z2, z3, z4])\n",
        "    \n",
        "    x = Dense(units=128, activation='selu')(x)\n",
        "    \n",
        "    x = Dense(units=1)(x)\n",
        "    x_output = ScaleLayer()(x)\n",
        "\n",
        "    model = Model(inputs=x_input, outputs=x_output, \n",
        "                  name='DNN_Model')\n",
        "    return model"
      ],
      "execution_count": 20,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "_d8ybRJc9co2",
        "outputId": "9f914336-4de8-465f-9e4a-f9283e0e6199"
      },
      "source": [
        "model = dnn_model()\n",
        "model.summary()"
      ],
      "execution_count": 21,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Model: \"DNN_Model\"\n",
            "__________________________________________________________________________________________________\n",
            "Layer (type)                    Output Shape         Param #     Connected to                     \n",
            "==================================================================================================\n",
            "input_1 (InputLayer)            [(None, 80, 64)]     0                                            \n",
            "__________________________________________________________________________________________________\n",
            "bidirectional (Bidirectional)   (None, 80, 1536)     5117952     input_1[0][0]                    \n",
            "__________________________________________________________________________________________________\n",
            "bidirectional_1 (Bidirectional) (None, 80, 1024)     8392704     bidirectional[0][0]              \n",
            "__________________________________________________________________________________________________\n",
            "bidirectional_2 (Bidirectional) (None, 80, 512)      2623488     bidirectional_1[0][0]            \n",
            "__________________________________________________________________________________________________\n",
            "bidirectional_4 (Bidirectional) (None, 80, 512)      1969152     bidirectional_1[0][0]            \n",
            "__________________________________________________________________________________________________\n",
            "add (Add)                       (None, 80, 512)      0           bidirectional_2[0][0]            \n",
            "                                                                 bidirectional_4[0][0]            \n",
            "__________________________________________________________________________________________________\n",
            "bidirectional_3 (Bidirectional) (None, 80, 256)      656384      bidirectional_2[0][0]            \n",
            "__________________________________________________________________________________________________\n",
            "bidirectional_5 (Bidirectional) (None, 80, 256)      493056      add[0][0]                        \n",
            "__________________________________________________________________________________________________\n",
            "add_1 (Add)                     (None, 80, 256)      0           bidirectional_3[0][0]            \n",
            "                                                                 bidirectional_5[0][0]            \n",
            "__________________________________________________________________________________________________\n",
            "bidirectional_6 (Bidirectional) (None, 80, 128)      123648      add_1[0][0]                      \n",
            "__________________________________________________________________________________________________\n",
            "concatenate (Concatenate)       (None, 80, 1152)     0           bidirectional_3[0][0]            \n",
            "                                                                 bidirectional_4[0][0]            \n",
            "                                                                 bidirectional_5[0][0]            \n",
            "                                                                 bidirectional_6[0][0]            \n",
            "__________________________________________________________________________________________________\n",
            "dense (Dense)                   (None, 80, 128)      147584      concatenate[0][0]                \n",
            "__________________________________________________________________________________________________\n",
            "dense_1 (Dense)                 (None, 80, 1)        129         dense[0][0]                      \n",
            "__________________________________________________________________________________________________\n",
            "scale_layer (ScaleLayer)        (None, 80, 1)        0           dense_1[0][0]                    \n",
            "==================================================================================================\n",
            "Total params: 19,524,097\n",
            "Trainable params: 19,524,097\n",
            "Non-trainable params: 0\n",
            "__________________________________________________________________________________________________\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "e-M0ZMvC9co3",
        "outputId": "123a260e-a03e-4b34-df44-22464a42be7b"
      },
      "source": [
        "plot_model(\n",
        "    model, \n",
        "    to_file='Google_Brain_Keras_Model.png', \n",
        "    show_shapes=True,\n",
        "    show_layer_names=True\n",
        ")\n"
      ],
      "execution_count": 22,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "10088"
            ]
          },
          "metadata": {},
          "execution_count": 22
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "rxWQVBYm-0VK"
      },
      "source": [
        "del model\n",
        "gc.collect()"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "u1v9SMji9co3"
      },
      "source": [
        "#reset Keras Session\n",
        "def reset_keras():\n",
        "    sess = tf.compat.v1.keras.backend.get_session()\n",
        "    tf.compat.v1.keras.backend.clear_session()\n",
        "    sess.close()\n",
        "    sess = tf.compat.v1.keras.backend.get_session()\n",
        "\n",
        "    # use the same config as you used to create the session\n",
        "    config = tf.compat.v1.ConfigProto()\n",
        "    config.gpu_options.per_process_gpu_memory_fraction = 1\n",
        "    config.gpu_options.visible_device_list = \"0\"\n",
        "    tf.compat.v1.keras.backend.set_session(tf.compat.v1.Session(config=config))\n",
        "    gc.collect()\n",
        "\n",
        "reset_keras()"
      ],
      "execution_count": 23,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "qDsDMBnE9co3"
      },
      "source": [
        "train_pred = np.zeros(targets.shape)"
      ],
      "execution_count": 24,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "id": "SYJkCi1-9co3",
        "outputId": "80053560-a5b4-4fdb-e6e2-fe62b3e005e9"
      },
      "source": [
        "with tpu_strategy.scope():\n",
        "    \n",
        "    VERBOSE = 1\n",
        "    EPOCHS = 300\n",
        "    FOLDS = 7\n",
        "    if DEBUG:\n",
        "      EPOCHS = 100\n",
        "    test_preds = []\n",
        "    kf = KFold(n_splits=FOLDS, shuffle=True, random_state=2021)\n",
        "    \n",
        "    for fold, (train_idx, test_idx) in enumerate(kf.split(train, targets)):\n",
        "        X_train, X_valid = train[train_idx], train[test_idx]\n",
        "        y_train, y_valid = targets[train_idx], targets[test_idx]\n",
        "        \n",
        "        model = dnn_model()\n",
        "        model.compile(optimizer=\"adam\", loss=\"mae\")\n",
        "\n",
        "        lr = ReduceLROnPlateau(monitor=\"val_loss\", factor=0.75, \n",
        "                               patience=10, verbose=VERBOSE)\n",
        "        \n",
        "        chk_point = ModelCheckpoint(f'./Bidirect_LSTM_model_{fold+1}C.h5',\n",
        "                                    monitor='val_loss', verbose=VERBOSE, \n",
        "                                    save_best_only=True, mode='min')\n",
        "\n",
        "        es = EarlyStopping(monitor=\"val_loss\", patience=50, \n",
        "                           verbose=VERBOSE, mode=\"min\", \n",
        "                           restore_best_weights=True)\n",
        "        \n",
        "        model.fit(X_train, y_train, \n",
        "                  validation_data=(X_valid, y_valid), \n",
        "                  epochs=EPOCHS,\n",
        "                  verbose=VERBOSE,\n",
        "                  batch_size=BATCH_SIZE, \n",
        "                  callbacks=[lr, chk_point, es])\n",
        "        \n",
        "        y_true = y_valid.squeeze().reshape(-1, 1)\n",
        "        y_pred = model.predict(X_valid, batch_size=BATCH_SIZE).squeeze().reshape(-1, 1)\n",
        "        train_pred[test_idx] = y_pred.reshape(-1, 80)\n",
        "        score = mean_absolute_error(y_true, y_pred)\n",
        "        print(f\"Fold-{fold+1} | OOF Score: {score}\")\n",
        "        \n",
        "        test_preds.append(model.predict(test, batch_size=BATCH_SIZE).squeeze().reshape(-1, 1).squeeze())\n",
        "        \n",
        "        del X_train, X_valid, y_train, y_valid, model\n",
        "        reset_keras()\n",
        "        gc.collect()"
      ],
      "execution_count": 25,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 1/300\n",
            "17/17 [==============================] - 88s 3s/step - loss: 5.1225 - val_loss: 2.8301\n",
            "\n",
            "Epoch 00001: val_loss improved from inf to 2.83009, saving model to ./Bidirect_LSTM_model_1C.h5\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/keras/utils/generic_utils.py:497: CustomMaskWarning: Custom mask layers require a config and must override get_config. When loading, the custom mask layer must be passed to the custom_objects argument.\n",
            "  category=CustomMaskWarning)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 2/300\n",
            "17/17 [==============================] - 9s 515ms/step - loss: 2.1683 - val_loss: 1.6377\n",
            "\n",
            "Epoch 00002: val_loss improved from 2.83009 to 1.63772, saving model to ./Bidirect_LSTM_model_1C.h5\n",
            "Epoch 3/300\n",
            "17/17 [==============================] - 9s 510ms/step - loss: 1.5732 - val_loss: 1.5571\n",
            "\n",
            "Epoch 00003: val_loss improved from 1.63772 to 1.55706, saving model to ./Bidirect_LSTM_model_1C.h5\n",
            "Epoch 4/300\n",
            "17/17 [==============================] - 9s 516ms/step - loss: 1.2305 - val_loss: 1.1424\n",
            "\n",
            "Epoch 00004: val_loss improved from 1.55706 to 1.14240, saving model to ./Bidirect_LSTM_model_1C.h5\n",
            "Epoch 5/300\n",
            "17/17 [==============================] - 9s 511ms/step - loss: 0.9583 - val_loss: 0.8837\n",
            "\n",
            "Epoch 00005: val_loss improved from 1.14240 to 0.88374, saving model to ./Bidirect_LSTM_model_1C.h5\n",
            "Epoch 6/300\n",
            "17/17 [==============================] - 9s 510ms/step - loss: 0.7913 - val_loss: 0.7895\n",
            "\n",
            "Epoch 00006: val_loss improved from 0.88374 to 0.78946, saving model to ./Bidirect_LSTM_model_1C.h5\n",
            "Epoch 7/300\n",
            "17/17 [==============================] - 9s 514ms/step - loss: 0.7361 - val_loss: 0.8020\n",
            "\n",
            "Epoch 00007: val_loss did not improve from 0.78946\n",
            "Epoch 8/300\n",
            "17/17 [==============================] - 9s 511ms/step - loss: 0.6998 - val_loss: 0.6485\n",
            "\n",
            "Epoch 00008: val_loss improved from 0.78946 to 0.64845, saving model to ./Bidirect_LSTM_model_1C.h5\n",
            "Epoch 9/300\n",
            "17/17 [==============================] - 9s 511ms/step - loss: 0.6823 - val_loss: 0.6571\n",
            "\n",
            "Epoch 00009: val_loss did not improve from 0.64845\n",
            "Epoch 10/300\n",
            "17/17 [==============================] - 9s 510ms/step - loss: 0.6233 - val_loss: 0.6029\n",
            "\n",
            "Epoch 00010: val_loss improved from 0.64845 to 0.60287, saving model to ./Bidirect_LSTM_model_1C.h5\n",
            "Epoch 11/300\n",
            "17/17 [==============================] - 9s 511ms/step - loss: 0.5958 - val_loss: 0.5670\n",
            "\n",
            "Epoch 00011: val_loss improved from 0.60287 to 0.56697, saving model to ./Bidirect_LSTM_model_1C.h5\n",
            "Epoch 12/300\n",
            "17/17 [==============================] - 9s 509ms/step - loss: 0.6246 - val_loss: 0.5919\n",
            "\n",
            "Epoch 00012: val_loss did not improve from 0.56697\n",
            "Epoch 13/300\n",
            "17/17 [==============================] - 9s 510ms/step - loss: 0.5661 - val_loss: 0.5397\n",
            "\n",
            "Epoch 00013: val_loss improved from 0.56697 to 0.53965, saving model to ./Bidirect_LSTM_model_1C.h5\n",
            "Epoch 14/300\n",
            "17/17 [==============================] - 9s 509ms/step - loss: 0.5470 - val_loss: 0.5453\n",
            "\n",
            "Epoch 00014: val_loss did not improve from 0.53965\n",
            "Epoch 15/300\n",
            "17/17 [==============================] - 9s 509ms/step - loss: 0.5116 - val_loss: 0.5249\n",
            "\n",
            "Epoch 00015: val_loss improved from 0.53965 to 0.52488, saving model to ./Bidirect_LSTM_model_1C.h5\n",
            "Epoch 16/300\n",
            "17/17 [==============================] - 9s 508ms/step - loss: 0.5182 - val_loss: 0.5096\n",
            "\n",
            "Epoch 00016: val_loss improved from 0.52488 to 0.50964, saving model to ./Bidirect_LSTM_model_1C.h5\n",
            "Epoch 17/300\n",
            "17/17 [==============================] - 9s 510ms/step - loss: 0.4890 - val_loss: 0.4781\n",
            "\n",
            "Epoch 00017: val_loss improved from 0.50964 to 0.47811, saving model to ./Bidirect_LSTM_model_1C.h5\n",
            "Epoch 18/300\n",
            "17/17 [==============================] - 9s 511ms/step - loss: 0.4768 - val_loss: 0.4705\n",
            "\n",
            "Epoch 00018: val_loss improved from 0.47811 to 0.47050, saving model to ./Bidirect_LSTM_model_1C.h5\n",
            "Epoch 19/300\n",
            "17/17 [==============================] - 9s 511ms/step - loss: 0.4892 - val_loss: 0.5280\n",
            "\n",
            "Epoch 00019: val_loss did not improve from 0.47050\n",
            "Epoch 20/300\n",
            "17/17 [==============================] - 9s 511ms/step - loss: 0.4750 - val_loss: 0.4583\n",
            "\n",
            "Epoch 00020: val_loss improved from 0.47050 to 0.45833, saving model to ./Bidirect_LSTM_model_1C.h5\n",
            "Epoch 21/300\n",
            "17/17 [==============================] - 9s 510ms/step - loss: 0.4847 - val_loss: 0.5885\n",
            "\n",
            "Epoch 00021: val_loss did not improve from 0.45833\n",
            "Epoch 22/300\n",
            "17/17 [==============================] - 9s 509ms/step - loss: 0.4855 - val_loss: 0.4849\n",
            "\n",
            "Epoch 00022: val_loss did not improve from 0.45833\n",
            "Epoch 23/300\n",
            "17/17 [==============================] - 9s 512ms/step - loss: 0.4754 - val_loss: 0.4673\n",
            "\n",
            "Epoch 00023: val_loss did not improve from 0.45833\n",
            "Epoch 24/300\n",
            "17/17 [==============================] - 9s 509ms/step - loss: 0.4560 - val_loss: 0.5281\n",
            "\n",
            "Epoch 00024: val_loss did not improve from 0.45833\n",
            "Epoch 25/300\n",
            "17/17 [==============================] - 9s 510ms/step - loss: 0.4410 - val_loss: 0.4832\n",
            "\n",
            "Epoch 00025: val_loss did not improve from 0.45833\n",
            "Epoch 26/300\n",
            "17/17 [==============================] - 9s 510ms/step - loss: 0.4551 - val_loss: 0.4740\n",
            "\n",
            "Epoch 00026: val_loss did not improve from 0.45833\n",
            "Epoch 27/300\n",
            "17/17 [==============================] - 9s 514ms/step - loss: 0.4416 - val_loss: 0.4667\n",
            "\n",
            "Epoch 00027: val_loss did not improve from 0.45833\n",
            "Epoch 28/300\n",
            "17/17 [==============================] - 9s 509ms/step - loss: 0.4139 - val_loss: 0.4503\n",
            "\n",
            "Epoch 00028: val_loss improved from 0.45833 to 0.45025, saving model to ./Bidirect_LSTM_model_1C.h5\n",
            "Epoch 29/300\n",
            "17/17 [==============================] - 9s 539ms/step - loss: 0.4080 - val_loss: 0.4503\n",
            "\n",
            "Epoch 00029: val_loss did not improve from 0.45025\n",
            "Epoch 30/300\n",
            "17/17 [==============================] - 9s 510ms/step - loss: 0.4024 - val_loss: 0.4239\n",
            "\n",
            "Epoch 00030: val_loss improved from 0.45025 to 0.42386, saving model to ./Bidirect_LSTM_model_1C.h5\n",
            "Epoch 31/300\n",
            "17/17 [==============================] - 9s 513ms/step - loss: 0.4138 - val_loss: 0.4419\n",
            "\n",
            "Epoch 00031: val_loss did not improve from 0.42386\n",
            "Epoch 32/300\n",
            "17/17 [==============================] - 9s 510ms/step - loss: 0.4052 - val_loss: 0.4398\n",
            "\n",
            "Epoch 00032: val_loss did not improve from 0.42386\n",
            "Epoch 33/300\n",
            "17/17 [==============================] - 9s 512ms/step - loss: 0.4002 - val_loss: 0.4407\n",
            "\n",
            "Epoch 00033: val_loss did not improve from 0.42386\n",
            "Epoch 34/300\n",
            "17/17 [==============================] - 9s 512ms/step - loss: 0.4121 - val_loss: 0.4738\n",
            "\n",
            "Epoch 00034: val_loss did not improve from 0.42386\n",
            "Epoch 35/300\n",
            "17/17 [==============================] - 9s 511ms/step - loss: 0.4031 - val_loss: 0.4117\n",
            "\n",
            "Epoch 00035: val_loss improved from 0.42386 to 0.41171, saving model to ./Bidirect_LSTM_model_1C.h5\n",
            "Epoch 36/300\n",
            "17/17 [==============================] - 9s 510ms/step - loss: 0.3875 - val_loss: 0.4283\n",
            "\n",
            "Epoch 00036: val_loss did not improve from 0.41171\n",
            "Epoch 37/300\n",
            "17/17 [==============================] - 9s 512ms/step - loss: 0.4104 - val_loss: 0.4735\n",
            "\n",
            "Epoch 00037: val_loss did not improve from 0.41171\n",
            "Epoch 38/300\n",
            "17/17 [==============================] - 9s 509ms/step - loss: 0.3888 - val_loss: 0.4691\n",
            "\n",
            "Epoch 00038: val_loss did not improve from 0.41171\n",
            "Epoch 39/300\n",
            "17/17 [==============================] - 9s 511ms/step - loss: 0.4029 - val_loss: 0.4625\n",
            "\n",
            "Epoch 00039: val_loss did not improve from 0.41171\n",
            "Epoch 40/300\n",
            "17/17 [==============================] - 9s 508ms/step - loss: 0.3764 - val_loss: 0.4281\n",
            "\n",
            "Epoch 00040: val_loss did not improve from 0.41171\n",
            "Epoch 41/300\n",
            "17/17 [==============================] - 9s 511ms/step - loss: 0.3658 - val_loss: 0.4243\n",
            "\n",
            "Epoch 00041: val_loss did not improve from 0.41171\n",
            "Epoch 42/300\n",
            "17/17 [==============================] - 9s 509ms/step - loss: 0.3810 - val_loss: 0.4228\n",
            "\n",
            "Epoch 00042: val_loss did not improve from 0.41171\n",
            "Epoch 43/300\n",
            "17/17 [==============================] - 9s 512ms/step - loss: 0.3508 - val_loss: 0.3888\n",
            "\n",
            "Epoch 00043: val_loss improved from 0.41171 to 0.38879, saving model to ./Bidirect_LSTM_model_1C.h5\n",
            "Epoch 44/300\n",
            "17/17 [==============================] - 9s 509ms/step - loss: 0.3494 - val_loss: 0.4183\n",
            "\n",
            "Epoch 00044: val_loss did not improve from 0.38879\n",
            "Epoch 45/300\n",
            "17/17 [==============================] - 9s 510ms/step - loss: 0.3744 - val_loss: 0.4180\n",
            "\n",
            "Epoch 00045: val_loss did not improve from 0.38879\n",
            "Epoch 46/300\n",
            "17/17 [==============================] - 9s 510ms/step - loss: 0.3498 - val_loss: 0.4073\n",
            "\n",
            "Epoch 00046: val_loss did not improve from 0.38879\n",
            "Epoch 47/300\n",
            "17/17 [==============================] - 9s 510ms/step - loss: 0.3445 - val_loss: 0.3877\n",
            "\n",
            "Epoch 00047: val_loss improved from 0.38879 to 0.38770, saving model to ./Bidirect_LSTM_model_1C.h5\n",
            "Epoch 48/300\n",
            "17/17 [==============================] - 9s 509ms/step - loss: 0.3397 - val_loss: 0.3766\n",
            "\n",
            "Epoch 00048: val_loss improved from 0.38770 to 0.37665, saving model to ./Bidirect_LSTM_model_1C.h5\n",
            "Epoch 49/300\n",
            "17/17 [==============================] - 9s 511ms/step - loss: 0.3345 - val_loss: 0.3808\n",
            "\n",
            "Epoch 00049: val_loss did not improve from 0.37665\n",
            "Epoch 50/300\n",
            "17/17 [==============================] - 9s 509ms/step - loss: 0.3414 - val_loss: 0.3991\n",
            "\n",
            "Epoch 00050: val_loss did not improve from 0.37665\n",
            "Epoch 51/300\n",
            "17/17 [==============================] - 9s 510ms/step - loss: 0.3433 - val_loss: 0.4536\n",
            "\n",
            "Epoch 00051: val_loss did not improve from 0.37665\n",
            "Epoch 52/300\n",
            "17/17 [==============================] - 9s 509ms/step - loss: 0.3794 - val_loss: 0.4385\n",
            "\n",
            "Epoch 00052: val_loss did not improve from 0.37665\n",
            "Epoch 53/300\n",
            "17/17 [==============================] - 9s 515ms/step - loss: 0.3310 - val_loss: 0.3733\n",
            "\n",
            "Epoch 00053: val_loss improved from 0.37665 to 0.37335, saving model to ./Bidirect_LSTM_model_1C.h5\n",
            "Epoch 54/300\n",
            "17/17 [==============================] - 9s 509ms/step - loss: 0.3140 - val_loss: 0.3941\n",
            "\n",
            "Epoch 00054: val_loss did not improve from 0.37335\n",
            "Epoch 55/300\n",
            "17/17 [==============================] - 9s 511ms/step - loss: 0.3103 - val_loss: 0.3718\n",
            "\n",
            "Epoch 00055: val_loss improved from 0.37335 to 0.37182, saving model to ./Bidirect_LSTM_model_1C.h5\n",
            "Epoch 56/300\n",
            "17/17 [==============================] - 9s 511ms/step - loss: 0.3332 - val_loss: 0.3913\n",
            "\n",
            "Epoch 00056: val_loss did not improve from 0.37182\n",
            "Epoch 57/300\n",
            "17/17 [==============================] - 9s 512ms/step - loss: 0.3321 - val_loss: 0.3922\n",
            "\n",
            "Epoch 00057: val_loss did not improve from 0.37182\n",
            "Epoch 58/300\n",
            "17/17 [==============================] - 9s 511ms/step - loss: 0.3172 - val_loss: 0.3624\n",
            "\n",
            "Epoch 00058: val_loss improved from 0.37182 to 0.36245, saving model to ./Bidirect_LSTM_model_1C.h5\n",
            "Epoch 59/300\n",
            "17/17 [==============================] - 9s 511ms/step - loss: 0.2978 - val_loss: 0.3839\n",
            "\n",
            "Epoch 00059: val_loss did not improve from 0.36245\n",
            "Epoch 60/300\n",
            "17/17 [==============================] - 9s 510ms/step - loss: 0.3119 - val_loss: 0.3771\n",
            "\n",
            "Epoch 00060: val_loss did not improve from 0.36245\n",
            "Epoch 61/300\n",
            "17/17 [==============================] - 9s 510ms/step - loss: 0.2994 - val_loss: 0.3647\n",
            "\n",
            "Epoch 00061: val_loss did not improve from 0.36245\n",
            "Epoch 62/300\n",
            "17/17 [==============================] - 9s 510ms/step - loss: 0.2875 - val_loss: 0.3667\n",
            "\n",
            "Epoch 00062: val_loss did not improve from 0.36245\n",
            "Epoch 63/300\n",
            "17/17 [==============================] - 9s 510ms/step - loss: 0.3007 - val_loss: 0.3882\n",
            "\n",
            "Epoch 00063: val_loss did not improve from 0.36245\n",
            "Epoch 64/300\n",
            "17/17 [==============================] - 9s 511ms/step - loss: 0.3018 - val_loss: 0.3719\n",
            "\n",
            "Epoch 00064: val_loss did not improve from 0.36245\n",
            "Epoch 65/300\n",
            "17/17 [==============================] - 9s 512ms/step - loss: 0.3167 - val_loss: 0.3982\n",
            "\n",
            "Epoch 00065: val_loss did not improve from 0.36245\n",
            "Epoch 66/300\n",
            "17/17 [==============================] - 9s 511ms/step - loss: 0.2919 - val_loss: 0.3596\n",
            "\n",
            "Epoch 00066: val_loss improved from 0.36245 to 0.35958, saving model to ./Bidirect_LSTM_model_1C.h5\n",
            "Epoch 67/300\n",
            "17/17 [==============================] - 9s 510ms/step - loss: 0.2777 - val_loss: 0.3687\n",
            "\n",
            "Epoch 00067: val_loss did not improve from 0.35958\n",
            "Epoch 68/300\n",
            "17/17 [==============================] - 9s 509ms/step - loss: 0.3071 - val_loss: 0.4100\n",
            "\n",
            "Epoch 00068: val_loss did not improve from 0.35958\n",
            "Epoch 69/300\n",
            "17/17 [==============================] - 9s 510ms/step - loss: 0.3017 - val_loss: 0.3586\n",
            "\n",
            "Epoch 00069: val_loss improved from 0.35958 to 0.35856, saving model to ./Bidirect_LSTM_model_1C.h5\n",
            "Epoch 70/300\n",
            "17/17 [==============================] - 9s 510ms/step - loss: 0.2744 - val_loss: 0.3594\n",
            "\n",
            "Epoch 00070: val_loss did not improve from 0.35856\n",
            "Epoch 71/300\n",
            "17/17 [==============================] - 9s 510ms/step - loss: 0.2978 - val_loss: 0.3531\n",
            "\n",
            "Epoch 00071: val_loss improved from 0.35856 to 0.35311, saving model to ./Bidirect_LSTM_model_1C.h5\n",
            "Epoch 72/300\n",
            "17/17 [==============================] - 9s 511ms/step - loss: 0.2648 - val_loss: 0.3652\n",
            "\n",
            "Epoch 00072: val_loss did not improve from 0.35311\n",
            "Epoch 73/300\n",
            "17/17 [==============================] - 9s 511ms/step - loss: 0.2836 - val_loss: 0.3708\n",
            "\n",
            "Epoch 00073: val_loss did not improve from 0.35311\n",
            "Epoch 74/300\n",
            "17/17 [==============================] - 9s 509ms/step - loss: 0.2703 - val_loss: 0.3771\n",
            "\n",
            "Epoch 00074: val_loss did not improve from 0.35311\n",
            "Epoch 75/300\n",
            "17/17 [==============================] - 9s 510ms/step - loss: 0.2723 - val_loss: 0.3546\n",
            "\n",
            "Epoch 00075: val_loss did not improve from 0.35311\n",
            "Epoch 76/300\n",
            "17/17 [==============================] - 9s 509ms/step - loss: 0.2718 - val_loss: 0.3879\n",
            "\n",
            "Epoch 00076: val_loss did not improve from 0.35311\n",
            "Epoch 77/300\n",
            "17/17 [==============================] - 9s 510ms/step - loss: 0.2738 - val_loss: 0.3488\n",
            "\n",
            "Epoch 00077: val_loss improved from 0.35311 to 0.34884, saving model to ./Bidirect_LSTM_model_1C.h5\n",
            "Epoch 78/300\n",
            "17/17 [==============================] - 9s 510ms/step - loss: 0.2646 - val_loss: 0.3477\n",
            "\n",
            "Epoch 00078: val_loss improved from 0.34884 to 0.34774, saving model to ./Bidirect_LSTM_model_1C.h5\n",
            "Epoch 79/300\n",
            "17/17 [==============================] - 9s 510ms/step - loss: 0.3060 - val_loss: 0.3985\n",
            "\n",
            "Epoch 00079: val_loss did not improve from 0.34774\n",
            "Epoch 80/300\n",
            "17/17 [==============================] - 9s 509ms/step - loss: 0.3157 - val_loss: 0.4113\n",
            "\n",
            "Epoch 00080: val_loss did not improve from 0.34774\n",
            "Epoch 81/300\n",
            "17/17 [==============================] - 9s 509ms/step - loss: 0.2809 - val_loss: 0.3466\n",
            "\n",
            "Epoch 00081: val_loss improved from 0.34774 to 0.34660, saving model to ./Bidirect_LSTM_model_1C.h5\n",
            "Epoch 82/300\n",
            "17/17 [==============================] - 9s 511ms/step - loss: 0.2579 - val_loss: 0.3842\n",
            "\n",
            "Epoch 00082: val_loss did not improve from 0.34660\n",
            "Epoch 83/300\n",
            "17/17 [==============================] - 9s 509ms/step - loss: 0.2618 - val_loss: 0.3451\n",
            "\n",
            "Epoch 00083: val_loss improved from 0.34660 to 0.34510, saving model to ./Bidirect_LSTM_model_1C.h5\n",
            "Epoch 84/300\n",
            "17/17 [==============================] - 9s 546ms/step - loss: 0.2720 - val_loss: 0.3936\n",
            "\n",
            "Epoch 00084: val_loss did not improve from 0.34510\n",
            "Epoch 85/300\n",
            "17/17 [==============================] - 9s 511ms/step - loss: 0.2625 - val_loss: 0.3465\n",
            "\n",
            "Epoch 00085: val_loss did not improve from 0.34510\n",
            "Epoch 86/300\n",
            "17/17 [==============================] - 9s 510ms/step - loss: 0.2471 - val_loss: 0.3403\n",
            "\n",
            "Epoch 00086: val_loss improved from 0.34510 to 0.34030, saving model to ./Bidirect_LSTM_model_1C.h5\n",
            "Epoch 87/300\n",
            "17/17 [==============================] - 9s 512ms/step - loss: 0.2515 - val_loss: 0.3451\n",
            "\n",
            "Epoch 00087: val_loss did not improve from 0.34030\n",
            "Epoch 88/300\n",
            "17/17 [==============================] - 9s 511ms/step - loss: 0.2356 - val_loss: 0.3446\n",
            "\n",
            "Epoch 00088: val_loss did not improve from 0.34030\n",
            "Epoch 89/300\n",
            "17/17 [==============================] - 9s 510ms/step - loss: 0.2417 - val_loss: 0.3331\n",
            "\n",
            "Epoch 00089: val_loss improved from 0.34030 to 0.33305, saving model to ./Bidirect_LSTM_model_1C.h5\n",
            "Epoch 90/300\n",
            "17/17 [==============================] - 9s 510ms/step - loss: 0.2315 - val_loss: 0.3450\n",
            "\n",
            "Epoch 00090: val_loss did not improve from 0.33305\n",
            "Epoch 91/300\n",
            "17/17 [==============================] - 9s 510ms/step - loss: 0.2308 - val_loss: 0.3333\n",
            "\n",
            "Epoch 00091: val_loss did not improve from 0.33305\n",
            "Epoch 92/300\n",
            "17/17 [==============================] - 9s 510ms/step - loss: 0.2295 - val_loss: 0.3401\n",
            "\n",
            "Epoch 00092: val_loss did not improve from 0.33305\n",
            "Epoch 93/300\n",
            "17/17 [==============================] - 9s 509ms/step - loss: 0.2350 - val_loss: 0.3417\n",
            "\n",
            "Epoch 00093: val_loss did not improve from 0.33305\n",
            "Epoch 94/300\n",
            "17/17 [==============================] - 9s 510ms/step - loss: 0.2509 - val_loss: 0.3437\n",
            "\n",
            "Epoch 00094: val_loss did not improve from 0.33305\n",
            "Epoch 95/300\n",
            "17/17 [==============================] - 9s 510ms/step - loss: 0.2414 - val_loss: 0.3432\n",
            "\n",
            "Epoch 00095: val_loss did not improve from 0.33305\n",
            "Epoch 96/300\n",
            "17/17 [==============================] - 9s 511ms/step - loss: 0.2622 - val_loss: 0.3527\n",
            "\n",
            "Epoch 00096: val_loss did not improve from 0.33305\n",
            "Epoch 97/300\n",
            "17/17 [==============================] - 9s 509ms/step - loss: 0.2490 - val_loss: 0.3338\n",
            "\n",
            "Epoch 00097: val_loss did not improve from 0.33305\n",
            "Epoch 98/300\n",
            "17/17 [==============================] - 9s 510ms/step - loss: 0.2238 - val_loss: 0.3407\n",
            "\n",
            "Epoch 00098: val_loss did not improve from 0.33305\n",
            "Epoch 99/300\n",
            "17/17 [==============================] - 9s 510ms/step - loss: 0.2333 - val_loss: 0.3294\n",
            "\n",
            "Epoch 00099: val_loss improved from 0.33305 to 0.32935, saving model to ./Bidirect_LSTM_model_1C.h5\n",
            "Epoch 100/300\n",
            " 7/17 [===========>..................] - ETA: 4s - loss: 0.2212"
          ]
        },
        {
          "output_type": "error",
          "ename": "KeyboardInterrupt",
          "evalue": "ignored",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-25-3ab7fc45a0f9>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m     28\u001b[0m                   \u001b[0mverbose\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mVERBOSE\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     29\u001b[0m                   \u001b[0mbatch_size\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mBATCH_SIZE\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 30\u001b[0;31m                   callbacks=[lr, chk_point, es])\n\u001b[0m\u001b[1;32m     31\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     32\u001b[0m         \u001b[0my_true\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0my_valid\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msqueeze\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mreshape\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m-\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/keras/engine/training.py\u001b[0m in \u001b[0;36mfit\u001b[0;34m(self, x, y, batch_size, epochs, verbose, callbacks, validation_split, validation_data, shuffle, class_weight, sample_weight, initial_epoch, steps_per_epoch, validation_steps, validation_batch_size, validation_freq, max_queue_size, workers, use_multiprocessing)\u001b[0m\n\u001b[1;32m   1187\u001b[0m               \u001b[0mlogs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtmp_logs\u001b[0m  \u001b[0;31m# No error, now safe to assign to logs.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1188\u001b[0m               \u001b[0mend_step\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mstep\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0mdata_handler\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstep_increment\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1189\u001b[0;31m               \u001b[0mcallbacks\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mon_train_batch_end\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mend_step\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlogs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1190\u001b[0m               \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstop_training\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1191\u001b[0m                 \u001b[0;32mbreak\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/keras/callbacks.py\u001b[0m in \u001b[0;36mon_train_batch_end\u001b[0;34m(self, batch, logs)\u001b[0m\n\u001b[1;32m    433\u001b[0m     \"\"\"\n\u001b[1;32m    434\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_should_call_train_batch_hooks\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 435\u001b[0;31m       \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_call_batch_hook\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mModeKeys\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mTRAIN\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'end'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbatch\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlogs\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mlogs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    436\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    437\u001b[0m   \u001b[0;32mdef\u001b[0m \u001b[0mon_test_batch_begin\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbatch\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlogs\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mNone\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/keras/callbacks.py\u001b[0m in \u001b[0;36m_call_batch_hook\u001b[0;34m(self, mode, hook, batch, logs)\u001b[0m\n\u001b[1;32m    293\u001b[0m       \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_call_batch_begin_hook\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmode\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbatch\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlogs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    294\u001b[0m     \u001b[0;32melif\u001b[0m \u001b[0mhook\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;34m'end'\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 295\u001b[0;31m       \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_call_batch_end_hook\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmode\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbatch\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlogs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    296\u001b[0m     \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    297\u001b[0m       \u001b[0;32mraise\u001b[0m \u001b[0mValueError\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'Unrecognized hook: {}'\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mformat\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mhook\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/keras/callbacks.py\u001b[0m in \u001b[0;36m_call_batch_end_hook\u001b[0;34m(self, mode, batch, logs)\u001b[0m\n\u001b[1;32m    313\u001b[0m       \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_batch_times\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mbatch_time\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    314\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 315\u001b[0;31m     \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_call_batch_hook_helper\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mhook_name\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbatch\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlogs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    316\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    317\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_batch_times\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m>=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_num_batches_for_timing_check\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/keras/callbacks.py\u001b[0m in \u001b[0;36m_call_batch_hook_helper\u001b[0;34m(self, hook_name, batch, logs)\u001b[0m\n\u001b[1;32m    351\u001b[0m     \u001b[0;32mfor\u001b[0m \u001b[0mcallback\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcallbacks\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    352\u001b[0m       \u001b[0mhook\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mgetattr\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mcallback\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mhook_name\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 353\u001b[0;31m       \u001b[0mhook\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mbatch\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlogs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    354\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    355\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_check_timing\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/keras/callbacks.py\u001b[0m in \u001b[0;36mon_train_batch_end\u001b[0;34m(self, batch, logs)\u001b[0m\n\u001b[1;32m   1026\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1027\u001b[0m   \u001b[0;32mdef\u001b[0m \u001b[0mon_train_batch_end\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbatch\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlogs\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mNone\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1028\u001b[0;31m     \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_batch_update_progbar\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mbatch\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlogs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1029\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1030\u001b[0m   \u001b[0;32mdef\u001b[0m \u001b[0mon_test_batch_end\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbatch\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlogs\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mNone\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/keras/callbacks.py\u001b[0m in \u001b[0;36m_batch_update_progbar\u001b[0;34m(self, batch, logs)\u001b[0m\n\u001b[1;32m   1098\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mverbose\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1099\u001b[0m       \u001b[0;31m# Only block async when verbose = 1.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1100\u001b[0;31m       \u001b[0mlogs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtf_utils\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msync_to_numpy_or_python_type\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mlogs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1101\u001b[0m       \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mprogbar\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mupdate\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mseen\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlist\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mlogs\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mitems\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfinalize\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mFalse\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1102\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/keras/utils/tf_utils.py\u001b[0m in \u001b[0;36msync_to_numpy_or_python_type\u001b[0;34m(tensors)\u001b[0m\n\u001b[1;32m    514\u001b[0m     \u001b[0;32mreturn\u001b[0m \u001b[0mt\u001b[0m  \u001b[0;31m# Don't turn ragged or sparse tensors to NumPy.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    515\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 516\u001b[0;31m   \u001b[0;32mreturn\u001b[0m \u001b[0mtf\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mnest\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmap_structure\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0m_to_single_numpy_or_python_type\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtensors\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    517\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    518\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/tensorflow/python/util/nest.py\u001b[0m in \u001b[0;36mmap_structure\u001b[0;34m(func, *structure, **kwargs)\u001b[0m\n\u001b[1;32m    867\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    868\u001b[0m   return pack_sequence_as(\n\u001b[0;32m--> 869\u001b[0;31m       \u001b[0mstructure\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0mfunc\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mx\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mentries\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    870\u001b[0m       expand_composites=expand_composites)\n\u001b[1;32m    871\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/tensorflow/python/util/nest.py\u001b[0m in \u001b[0;36m<listcomp>\u001b[0;34m(.0)\u001b[0m\n\u001b[1;32m    867\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    868\u001b[0m   return pack_sequence_as(\n\u001b[0;32m--> 869\u001b[0;31m       \u001b[0mstructure\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0mfunc\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mx\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mentries\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    870\u001b[0m       expand_composites=expand_composites)\n\u001b[1;32m    871\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/keras/utils/tf_utils.py\u001b[0m in \u001b[0;36m_to_single_numpy_or_python_type\u001b[0;34m(t)\u001b[0m\n\u001b[1;32m    510\u001b[0m   \u001b[0;32mdef\u001b[0m \u001b[0m_to_single_numpy_or_python_type\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mt\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    511\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0misinstance\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mt\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtf\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mTensor\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 512\u001b[0;31m       \u001b[0mx\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mt\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mnumpy\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    513\u001b[0m       \u001b[0;32mreturn\u001b[0m \u001b[0mx\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mitem\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mndim\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;36m0\u001b[0m \u001b[0;32melse\u001b[0m \u001b[0mx\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    514\u001b[0m     \u001b[0;32mreturn\u001b[0m \u001b[0mt\u001b[0m  \u001b[0;31m# Don't turn ragged or sparse tensors to NumPy.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/tensorflow/python/framework/ops.py\u001b[0m in \u001b[0;36mnumpy\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m   1092\u001b[0m     \"\"\"\n\u001b[1;32m   1093\u001b[0m     \u001b[0;31m# TODO(slebedev): Consider avoiding a copy for non-CPU or remote tensors.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1094\u001b[0;31m     \u001b[0mmaybe_arr\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_numpy\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m  \u001b[0;31m# pylint: disable=protected-access\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1095\u001b[0m     \u001b[0;32mreturn\u001b[0m \u001b[0mmaybe_arr\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcopy\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0misinstance\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmaybe_arr\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mndarray\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32melse\u001b[0m \u001b[0mmaybe_arr\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1096\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/tensorflow/python/framework/ops.py\u001b[0m in \u001b[0;36m_numpy\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m   1058\u001b[0m   \u001b[0;32mdef\u001b[0m \u001b[0m_numpy\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1059\u001b[0m     \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1060\u001b[0;31m       \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_numpy_internal\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1061\u001b[0m     \u001b[0;32mexcept\u001b[0m \u001b[0mcore\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_NotOkStatusException\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m:\u001b[0m  \u001b[0;31m# pylint: disable=protected-access\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1062\u001b[0m       \u001b[0msix\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mraise_from\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mcore\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_status_to_exception\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0me\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcode\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmessage\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m)\u001b[0m  \u001b[0;31m# pylint: disable=protected-access\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "fUanGPDq9co3"
      },
      "source": [
        "train_pred = train_pred.flatten()\n",
        "targets = targets.flatten()\n",
        "score = mean_absolute_error(targets, train_pred)\n",
        "print(f\"Train Score: {score}\")"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "QHx2oF849co4"
      },
      "source": [
        "## Create submission file"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "tS9VjV7K9co4"
      },
      "source": [
        "submission = pd.read_csv('../input/ventilator-pressure-prediction/sample_submission.csv')\n",
        "submission[\"pressure\"] = sum(test_preds)/5\n",
        "submission.to_csv('submission.csv', index=False)\n",
        "\n",
        "# ENSEMBLE FOLDS WITH MEDIAN\n",
        "#取中位数\n",
        "submission[\"pressure\"] = np.median(np.vstack(test_preds),axis=0)\n",
        "submission.to_csv('submission_median.csv', index=False)\n",
        "\n",
        "\n",
        "# ENSEMBLE FOLDS WITH MEDIAN AND ROUND PREDICTIONS\n",
        "submission[\"pressure\"] =\\\n",
        "    np.round( (submission.pressure - pressure_min)/pressure_step ) * pressure_step + pressure_min\n",
        "submission.pressure = np.clip(submission.pressure, pressure_min, pressure_max)\n",
        "submission.to_csv('submission_median_round.csv', index=False)"
      ]
    }
  ]
}