gquest-bert-base.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "/opt/conda/lib/python3.6/site-packages/pip/_internal/commands/install.py:243: UserWarning: Disabling all use of wheels due to the use of --build-options / --global-options / --install-options.\r\n",
      "  cmdoptions.check_install_build_global(options)\r\n",
      "Processing /kaggle/input/nvidia-apex/apex-880ab92\r\n",
      "Skipping bdist_wheel for apex, due to binaries being disabled for it.\r\n",
      "Installing collected packages: apex\r\n",
      "  Running setup.py install for apex ... \u001b[?25l-\b \b\\\b \b|\b \b/\b \b-\b \b\\\b \b|\b \b/\b \b-\b \b\\\b \b|\b \b/\b \b-\b \b\\\b \b|\b \b/\b \b-\b \b\\\b \b|\b \b/\b \b-\b \bdone\r\n",
      "\u001b[?25hSuccessfully installed apex-0.1\r\n"
     ]
    }
   ],
   "source": [
    "# Install from kaggle datasets for code competition\n",
    "\n",
    "# Install transformers 2.3.0\n",
    "! pip install -q ../input/sacremoses/sacremoses-master/\n",
    "! pip install -q ../input/transformers-2-3-0/\n",
    "\n",
    "# Install Nvidia Apex\n",
    "! pip install -q -v --no-cache-dir --global-option=\"--cpp_ext\" --global-option=\"--cuda_ext\" /kaggle/input/nvidia-apex/apex-880ab92"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "torch version: 1.2.0\n",
      "transformers version: 2.3.0\n"
     ]
    }
   ],
   "source": [
    "from __future__ import absolute_import, division, print_function\n",
    "\n",
    "import os, sys, re, gc, pickle, operator, shutil, copy, random\n",
    "import time, datetime\n",
    "from collections import namedtuple\n",
    "\n",
    "from math import floor, ceil\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "from scipy.stats import spearmanr\n",
    "from sklearn.model_selection import KFold, StratifiedKFold, GroupKFold\n",
    "from sklearn.metrics import accuracy_score, roc_auc_score\n",
    "from sklearn.utils import shuffle\n",
    "\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.nn.functional as F\n",
    "import torch.optim as optim\n",
    "from torch.utils.data import TensorDataset, Dataset, DataLoader, Sampler\n",
    "\n",
    "from apex import amp\n",
    "import transformers\n",
    "from transformers import *\n",
    "\n",
    "import matplotlib\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "\n",
    "%matplotlib inline\n",
    "\n",
    "print(f'torch version: {torch.__version__}')\n",
    "print(f'transformers version: {transformers.__version__}')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Configuration"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "DATA_DIR = \"/kaggle/input/google-quest-challenge/\"\n",
    "VOCAB_PATH = '/kaggle/input/bert-pretrained-models/uncased_l-12_h-768_a-12/uncased_L-12_H-768_A-12/'\n",
    "MODEL_PATH = '/kaggle/input/bert-pytorch/'\n",
    "output_model_file = \"quest_bert_models.pt\"\n",
    "\n",
    "SEED = 2019\n",
    "device = torch.device('cuda') if torch.cuda.is_available() else torch.device('cpu')\n",
    "\n",
    "MAX_SEQUENCE_LENGTH = 512\n",
    "batch_size = 8\n",
    "grad_accumulation_steps = 2\n",
    "\n",
    "epochs_for_sched = 3\n",
    "checkpoint_iter = 470\n",
    "lr = 1e-4\n",
    "warmup_proportion = 0.2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Seed for randomness in pytorch\n",
    "def seed_everything(seed=SEED):\n",
    "    random.seed(seed)\n",
    "    np.random.seed(seed)\n",
    "    torch.manual_seed(seed)\n",
    "    torch.cuda.manual_seed(seed)\n",
    "    torch.backends.cudnn.deterministic = True\n",
    "    torch.backends.cudnn.benchmark = True\n",
    "\n",
    "seed_everything()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Datasets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "def trim_and_tokenize(title, question, answer, max_sequence_length, tokenizer,\n",
    "                      trunc_mode='head', t_max_len=18, q_max_len=245, a_max_len=244):\n",
    "    \"\"\"\n",
    "    trunc_mode:\n",
    "    - head: truncate sequence from head\n",
    "    - tail: truncate sequence from tail\n",
    "    - mix: concatenate truncated sequences from head and tail 6:4 for each\n",
    "    \"\"\"\n",
    "    \n",
    "    assert trunc_mode in {\"head\", \"tail\", \"mix\"}\n",
    "    need_trunc = False\n",
    "\n",
    "    tq_sep = tokenizer.tokenize(\"Details:\")\n",
    "    t = tokenizer.tokenize(title)\n",
    "    q = tokenizer.tokenize(question)\n",
    "    a = tokenizer.tokenize(answer)\n",
    "    \n",
    "    t_len = len(t)\n",
    "    q_len = len(q)\n",
    "    a_len = len(a)\n",
    "\n",
    "    if (t_len+q_len+a_len+5) > max_sequence_length:\n",
    "        need_trunc = True\n",
    "        \n",
    "        if t_max_len > t_len:\n",
    "            t_new_len = t_len\n",
    "            a_max_len = a_max_len + floor((t_max_len - t_len)/2)\n",
    "            q_max_len = q_max_len + ceil((t_max_len - t_len)/2)\n",
    "        else:\n",
    "            t_new_len = t_max_len\n",
    "      \n",
    "        if a_max_len > a_len:\n",
    "            a_new_len = a_len \n",
    "            q_new_len = q_max_len + (a_max_len - a_len)\n",
    "        elif q_max_len > q_len:\n",
    "            a_new_len = a_max_len + (q_max_len - q_len)\n",
    "            q_new_len = q_len\n",
    "        else:\n",
    "            a_new_len = a_max_len\n",
    "            q_new_len = q_max_len\n",
    "            \n",
    "        if t_new_len+a_new_len+q_new_len+5 != max_sequence_length:\n",
    "            raise ValueError(\"New sequence length should be %d, but is %d\" \n",
    "                             % (max_sequence_length, (t_new_len+a_new_len+q_new_len+5)))\n",
    "        \n",
    "        if trunc_mode == \"head\":\n",
    "            t = t[:t_new_len]\n",
    "            q = q[:q_new_len]\n",
    "            a = a[:a_new_len]\n",
    "        if trunc_mode == \"tail\":\n",
    "            t = t[-t_new_len:]\n",
    "            q = q[-q_new_len:]\n",
    "            a = a[-a_new_len:]\n",
    "        if trunc_mode == \"mix\":\n",
    "            def trunc_seq(seq, seq_max_len, trunc_ratio=0.6):\n",
    "                maj_len = int(seq_max_len * trunc_ratio)\n",
    "                return seq[:maj_len] + seq[-(seq_max_len-maj_len):]\n",
    "            t = trunc_seq(t, t_new_len)\n",
    "            q = trunc_seq(q, q_new_len)\n",
    "            a = trunc_seq(a, a_new_len)\n",
    "    \n",
    "    return t, tq_sep, q, a, need_trunc\n",
    "\n",
    "\n",
    "# Tokenizing the lines to BERT token\n",
    "def convert_lines(df, columns, max_sequence_length, tokenizer, trunc_mode='head',\n",
    "                  misc_trunc=False, target=None, sample_weighting=False):\n",
    "    \"\"\"\n",
    "    trunc_mode:\n",
    "    - head: truncate sequence from head\n",
    "    - tail: truncate sequence from tail\n",
    "    - mix: concatenate truncated sequences from head and tail 6:4 for each\n",
    "    \n",
    "    misc_trunc: use miscellaneous truncation or not\n",
    "    \n",
    "    sample_weighting: when misc_trunc=True, weighting on different truncated samples\n",
    "    \"\"\"\n",
    "    \n",
    "    all_tokens = []\n",
    "    segment_ids = []   # representing segmentation of sentence A and B\n",
    "    if target is not None:\n",
    "        labels = []\n",
    "    \n",
    "    for ind, (_, instance) in enumerate(df[columns].iterrows()):\n",
    "        \n",
    "        title, question, answer = instance.question_title, instance.question_body, instance.answer\n",
    "        t, tq_sep, q, a, need_trunc = trim_and_tokenize(title, question, answer,\n",
    "                                                        max_sequence_length, tokenizer, trunc_mode=trunc_mode)\n",
    "        tokens = [\"[CLS]\"] + t + tq_sep + q + [\"[SEP]\"] + a + [\"[SEP]\"]\n",
    "        all_tokens.append(tokenizer.convert_tokens_to_ids(tokens))\n",
    "        segment_ids.append([0]*(len(t)+len(tq_sep)+len(q)+2) + [1]*(len(a)+1))\n",
    "        if target is not None:\n",
    "            if sample_weighting:\n",
    "                labels.append(np.concatenate([target[ind], [3./7]]))\n",
    "            else:\n",
    "                labels.append(target[ind])\n",
    "        \n",
    "        if need_trunc and misc_trunc:\n",
    "            t, tq_sep, q, a, _ = trim_and_tokenize(title, question, answer,\n",
    "                                                   max_sequence_length, tokenizer, trunc_mode='tail')\n",
    "            tokens = [\"[CLS]\"] + t + tq_sep + q + [\"[SEP]\"] + a + [\"[SEP]\"]\n",
    "            all_tokens.append(tokenizer.convert_tokens_to_ids(tokens))\n",
    "            segment_ids.append([0]*(len(t)+len(tq_sep)+len(q)+2) + [1]*(len(a)+1))\n",
    "\n",
    "            t, tq_sep, q, a, _ = trim_and_tokenize(title, question, answer,\n",
    "                                                   max_sequence_length, tokenizer, trunc_mode='mix')\n",
    "            tokens = [\"[CLS]\"] + t + tq_sep + q + [\"[SEP]\"] + a + [\"[SEP]\"]\n",
    "            all_tokens.append(tokenizer.convert_tokens_to_ids(tokens))\n",
    "            segment_ids.append([0]*(len(t)+len(tq_sep)+len(q)+2) + [1]*(len(a)+1))\n",
    "\n",
    "            if target is not None:\n",
    "                if sample_weighting:\n",
    "                    labels.extend([np.concatenate([target[ind], [2./7]])] * 2)\n",
    "                else:\n",
    "                    labels.extend([target[ind]] * 2)\n",
    "    \n",
    "    if target is not None:\n",
    "        return np.array(all_tokens), np.array(segment_ids), np.array(labels)\n",
    "    return np.array(all_tokens), np.array(segment_ids)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Prepare dataset and dataloader\n",
    "\n",
    "class QuestQAs(Dataset):\n",
    "\n",
    "    def __init__(self, tokenized_comments, segment_ids, targets=None, split=None, maxlen=256):\n",
    "        self.comments = tokenized_comments\n",
    "        self.segment_ids = segment_ids\n",
    "        self.targets = targets\n",
    "        self.split = split\n",
    "        assert self.split in {'train', 'valid', 'test'}\n",
    "        self.maxlen = maxlen\n",
    "\n",
    "    def __getitem__(self, index):\n",
    "        comment = self.comments[index]\n",
    "        segment_id = self.segment_ids[index]\n",
    "        if self.targets is not None:\n",
    "            target = self.targets[index]\n",
    "            return comment, segment_id, torch.FloatTensor(target)\n",
    "        else:\n",
    "            return comment, segment_id\n",
    "\n",
    "    def __len__(self):\n",
    "        return len(self.comments)\n",
    "\n",
    "    def get_lens(self):\n",
    "        lengths = np.fromiter(\n",
    "            ((min(self.maxlen, len(seq))) for seq in self.comments),\n",
    "            dtype=np.int32)\n",
    "        return lengths\n",
    "\n",
    "    def collate_fn(self, batch):\n",
    "        \"\"\"\n",
    "        Collate function for sequence bucketing\n",
    "        Note: this need not be defined in this Class, can be standalone.\n",
    "\n",
    "        :param batch: an iterable of N sets from __getitem__()\n",
    "        :return: a tensor of comments, and targets\n",
    "        \"\"\"\n",
    "\n",
    "        if self.split in ('train', 'valid'):\n",
    "            comments, segment_ids, targets = zip(*batch)\n",
    "        else:\n",
    "            comments, segment_ids = zip(*batch)\n",
    "\n",
    "        lengths = [len(c) for c in comments]\n",
    "        maxlen = max(lengths)\n",
    "        padded_comments, padded_seg_ids = [], []\n",
    "        for i, (c, s) in enumerate(zip(comments, segment_ids)):\n",
    "            padded_comments.append(c+[0]*(maxlen - lengths[i]))\n",
    "            padded_seg_ids.append(s +[0]*(maxlen - lengths[i]))\n",
    "\n",
    "        if self.split in ('train', 'valid'):\n",
    "            return torch.LongTensor(padded_comments), torch.LongTensor(padded_seg_ids), torch.stack(targets)\n",
    "        else:\n",
    "            return torch.LongTensor(padded_comments), torch.LongTensor(padded_seg_ids)\n",
    "\n",
    "\n",
    "class BucketSampler(Sampler):\n",
    "\n",
    "    def __init__(self, data_source, sort_lens, bucket_size=None, batch_size=1024, shuffle_data=True):\n",
    "        super().__init__(data_source)\n",
    "        self.shuffle = shuffle_data\n",
    "        self.batch_size = batch_size\n",
    "        self.sort_lens = sort_lens\n",
    "        self.bucket_size = bucket_size if bucket_size is not None else len(sort_lens)\n",
    "        self.weights = None\n",
    "\n",
    "        if not shuffle_data:\n",
    "            self.index = self.prepare_buckets()\n",
    "        else:\n",
    "            self.index = None\n",
    "\n",
    "    def set_weights(self, weights):\n",
    "        assert weights >= 0\n",
    "        total = np.sum(weights)\n",
    "        if total != 1:\n",
    "            weights = weights / total\n",
    "        self.weights = weights\n",
    "\n",
    "    def __iter__(self):\n",
    "        indices = None\n",
    "        if self.weights is not None:\n",
    "            total = len(self.sort_lens)\n",
    "            indices = np.random.choice(total, (total,), p=self.weights)\n",
    "        if self.shuffle:\n",
    "            self.index = self.prepare_buckets(indices)\n",
    "        return iter(self.index)\n",
    "\n",
    "    def get_reverse_indexes(self):\n",
    "        indexes = np.zeros((len(self.index),), dtype=np.int32)\n",
    "        for i, j in enumerate(self.index):\n",
    "            indexes[j] = i\n",
    "        return indexes\n",
    "\n",
    "    def __len__(self):\n",
    "        return len(self.sort_lens)\n",
    "\n",
    "    def prepare_buckets(self, indices=None):\n",
    "        lengths = - self.sort_lens\n",
    "        assert self.bucket_size % self.batch_size == 0 or self.bucket_size == len(lengths)\n",
    "\n",
    "        if indices is None:\n",
    "            if self.shuffle:\n",
    "                indices = shuffle(np.arange(len(lengths), dtype=np.int32))\n",
    "                lengths = lengths[indices]\n",
    "            else:\n",
    "                indices = np.arange(len(lengths), dtype=np.int32)\n",
    "\n",
    "        #  bucket iterator\n",
    "        def divide_chunks(l, n):\n",
    "            if n == len(l):\n",
    "                yield np.arange(len(l), dtype=np.int32), l\n",
    "            else:\n",
    "                # looping till length l\n",
    "                for i in range(0, len(l), n):\n",
    "                    data = l[i:i + n]\n",
    "                    yield np.arange(i, i + len(data), dtype=np.int32), data\n",
    "\n",
    "        new_indices = []\n",
    "        extra_batch_idx = None\n",
    "        for chunk_index, chunk in divide_chunks(lengths, self.bucket_size):\n",
    "            # sort indices in bucket by descending order of length\n",
    "            indices_sorted = chunk_index[np.argsort(chunk)]\n",
    "\n",
    "            batch_idxes = []\n",
    "            for _, batch_idx in divide_chunks(indices_sorted, self.batch_size):\n",
    "                if len(batch_idx) == self.batch_size:\n",
    "                    batch_idxes.append(batch_idx.tolist())\n",
    "                else:\n",
    "                    assert extra_batch_idx is None\n",
    "                    assert batch_idx is not None\n",
    "                    extra_batch_idx = batch_idx.tolist()\n",
    "\n",
    "            # shuffling batches within buckets\n",
    "            if self.shuffle:\n",
    "                batch_idxes = shuffle(batch_idxes)\n",
    "            for batch_idx in batch_idxes:\n",
    "                new_indices.extend(batch_idx)\n",
    "\n",
    "        if extra_batch_idx is not None:\n",
    "            new_indices.extend(extra_batch_idx)\n",
    "\n",
    "        if not self.shuffle:\n",
    "            self.original_indices = np.argsort(indices_sorted).tolist()\n",
    "        return indices[new_indices]\n",
    "\n",
    "\n",
    "def prepare_loader(x, seg_ids, y=None, batch_size=None, split=None):\n",
    "    assert split in {'train', 'valid', 'test'}\n",
    "    dataset = QuestQAs(x, seg_ids, y, split, MAX_SEQUENCE_LENGTH)\n",
    "    if split == 'train':\n",
    "        sampler = BucketSampler(dataset, dataset.get_lens(),\n",
    "                                bucket_size=batch_size*20, batch_size=batch_size)\n",
    "        return DataLoader(dataset, batch_size=batch_size, sampler=sampler,\n",
    "                          collate_fn=dataset.collate_fn)\n",
    "    else:\n",
    "        sampler = BucketSampler(dataset, dataset.get_lens(),\n",
    "                                batch_size=batch_size, shuffle_data=False)\n",
    "        return DataLoader(dataset, batch_size=batch_size, sampler=sampler,\n",
    "                          collate_fn=dataset.collate_fn), sampler.original_indices"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "def train_val_split(train_df):\n",
    "    kf = GroupKFold(n_splits=5)\n",
    "#     kf = KFold(n_splits=5, shuffle=True, random_state=SEED)\n",
    "    cv_indices = list(kf.split(train_df.question_body, groups=train_df.question_body))\n",
    "#     cv_indices = list(kf.split(train_df))\n",
    "    return cv_indices\n",
    "\n",
    "\n",
    "def load_data():\n",
    "    train_df = pd.read_csv(DATA_DIR+'train.csv')\n",
    "    output_cols = list(train_df.columns[11:])\n",
    "    input_cols = list(train_df.columns[[1,2,5]])\n",
    "    \n",
    "    train_tars = train_df[output_cols].values.astype('float32')\n",
    "    \n",
    "    return train_tars, train_df, input_cols\n",
    "\n",
    "\n",
    "def load_and_preproc():\n",
    "    train_df = pd.read_csv(DATA_DIR+'train.csv')\n",
    "    output_cols = list(train_df.columns[11:])\n",
    "    input_cols = list(train_df.columns[[1,2,5]])\n",
    "    \n",
    "    train_tars = train_df[output_cols].values.astype('float32')\n",
    "    train_seq, train_seg_ids = convert_lines(train_df, input_cols, MAX_SEQUENCE_LENGTH, tokenizer)\n",
    "\n",
    "    return train_seq, train_seg_ids, train_tars, train_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loading and tokenizing...\n",
      "tokenizing complete in 0 seconds.\n"
     ]
    }
   ],
   "source": [
    "tokenizer = BertTokenizer.from_pretrained(VOCAB_PATH + 'vocab.txt')\n",
    "\n",
    "t0 = time.time()\n",
    "print('Loading and tokenizing...')\n",
    "train_tars, train_df, input_cols = load_data()\n",
    "# train_seq, train_seg_ids, train_tars, train_df = load_and_preproc()\n",
    "cv_indices = train_val_split(train_df)\n",
    "print('tokenizing complete in {:.0f} seconds.'.format(time.time()-t0))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Classifier\n",
    "class GQuestNet(BertPreTrainedModel):\n",
    "    def __init__(self, config, num_labels):\n",
    "        super(GQuestNet, self).__init__(config)\n",
    "        self.bert = BertModel.from_pretrained(MODEL_PATH+'bert-base-uncased-pytorch_model.bin', config=config)\n",
    "        self.dropouts = nn.ModuleList([nn.Dropout(0.3) for _ in range(5)])\n",
    "        self.classifier = nn.Linear(config.hidden_size*4, num_labels)\n",
    "\n",
    "    def forward(self, input_ids, attention_mask=None, token_type_ids=None):\n",
    "        _, _, encoded_layers = self.bert(input_ids, attention_mask, token_type_ids)\n",
    "        seq_op1 = encoded_layers[-1]\n",
    "        seq_op2 = encoded_layers[-2]\n",
    "        avg_pool1 = torch.mean(seq_op1, 1)\n",
    "        max_pool1, _ = torch.max(seq_op1, 1)\n",
    "        avg_pool2 = torch.mean(seq_op2, 1)\n",
    "        max_pool2, _ = torch.max(seq_op2, 1)\n",
    "        pooled_output = torch.cat((avg_pool1, max_pool1, avg_pool2, max_pool2), 1)\n",
    "        for i, dropout in enumerate(self.dropouts):\n",
    "            if i == 0:\n",
    "                h = self.classifier(dropout(pooled_output))\n",
    "            else:\n",
    "                h += self.classifier(dropout(pooled_output))\n",
    "        return h / len(self.dropouts)\n",
    "\n",
    "\n",
    "class TruncLoss(nn.Module):\n",
    "    def forward(self, pred_scores, labels):\n",
    "        loss = 0\n",
    "        for i in range(pred_scores.size(1)):\n",
    "            loss += nn.BCEWithLogitsLoss(weight=labels[:,-1])(pred_scores[:,i], labels[:,i])\n",
    "        return loss\n",
    "\n",
    "\n",
    "# Build model and optimizer\n",
    "def model_optimizer_init(ft_lrs, num_labels=30):\n",
    "    print(\"Building model and optimizer...\")\n",
    "    cfg = BertConfig.from_pretrained(MODEL_PATH + 'bert-base-uncased-config.json')\n",
    "    cfg.output_hidden_states = True\n",
    "    model = GQuestNet(cfg, num_labels=num_labels)\n",
    "\n",
    "    params_bert = list(model.bert.parameters())\n",
    "    params_cls = list(model.classifier.parameters())\n",
    "\n",
    "    optimizer_grouped_parameters = [\n",
    "        {'params': params_bert, 'lr':ft_lrs[0]},\n",
    "        {'params': params_cls, 'lr':ft_lrs[1]}\n",
    "        ]\n",
    "    \n",
    "    optimizer = optim.AdamW(optimizer_grouped_parameters)\n",
    "    \n",
    "    return model, optimizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "torch.Size([8, 30])\n"
     ]
    }
   ],
   "source": [
    "def model_test():\n",
    "    x = torch.randint(10, (8, 256))\n",
    "    cfg = BertConfig.from_pretrained(MODEL_PATH + 'bert-base-uncased-config.json')\n",
    "    cfg.output_hidden_states = True\n",
    "    model = GQuestNet(cfg, num_labels=30)\n",
    "    print(model(x).size())\n",
    "\n",
    "model_test()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Solver"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "_kg_hide-input": false
   },
   "outputs": [],
   "source": [
    "# Compute metrics\n",
    "def compute_rho(labels, preds):\n",
    "    rhos = []\n",
    "    for col_label, col_pred in zip(labels.T, preds.T):\n",
    "        rhos.append(\n",
    "            spearmanr(col_label, col_pred + np.random.normal(0, 1e-7, col_pred.shape[0])).correlation)\n",
    "    return np.mean(rhos)\n",
    "\n",
    "\n",
    "# Functions for the training process\n",
    "class NetSolver(object):\n",
    "\n",
    "    def __init__(self, model, criterion, optimizer, scheduler, print_freq, filepath):\n",
    "        self.model = model\n",
    "        self.criterion = criterion\n",
    "        self.optimizer = optimizer\n",
    "        self.scheduler = scheduler\n",
    "        self.print_freq = print_freq\n",
    "        self.filepath = filepath\n",
    "\n",
    "        self.model = self.model.to(device)\n",
    "        self.criterion = self.criterion.to(device)\n",
    "        self._reset()\n",
    "\n",
    "    def _reset(self):\n",
    "        \"\"\"Set up some book-keeping variables for optimization.\n",
    "        \"\"\"\n",
    "        self.best_val_loss = 1e4\n",
    "        self.best_val_rho = 0.\n",
    "        self.loss_history = []\n",
    "        self.val_loss_history = []\n",
    "        self.rho_history = []\n",
    "        self.val_rho_history = []\n",
    "        self.val_preds = []\n",
    "        self.models = {}\n",
    "\n",
    "    def save_checkpoint(self, iteration):\n",
    "        \"\"\"Save model checkpoint.\n",
    "        \"\"\"\n",
    "#         self.models[f'ckpt_{iteration}'] = self.model.state_dict()\n",
    "        self.models['ckpt_best'] = self.model.state_dict()\n",
    "    \n",
    "    def save_final_state(self):\n",
    "        \"\"\"Save final states.\n",
    "        \"\"\"\n",
    "        state = {'model': self.models,\n",
    "#                  'optimizer': self.optimizer.state_dict()\n",
    "                 'optimizer': None\n",
    "                 }\n",
    "        torch.save(state, self.filepath)\n",
    "\n",
    "    def forward_pass(self, x, seg_ids, y):\n",
    "        \"\"\"Forward pass through the network.\n",
    "        \"\"\"\n",
    "        x, y = x.to(device=device, dtype=torch.long), y.to(device=device, dtype=torch.float)\n",
    "        seg_ids = seg_ids.to(device=device, dtype=torch.long)\n",
    "        scores = self.model(x, token_type_ids=seg_ids, attention_mask=(x>0))\n",
    "        loss = self.criterion(scores, y)\n",
    "        return loss, torch.sigmoid(scores)\n",
    "\n",
    "    def train(self, loaders, iterations, start_time):\n",
    "        \"\"\"Weight of network updated by apex, grad accumulation, model checkpoint.\n",
    "        \"\"\"\n",
    "        train_loader, val_loader = loaders\n",
    "        loader = iter(train_loader)\n",
    "        running_loss = 0.\n",
    "        self.optimizer.zero_grad()\n",
    "\n",
    "        # start training for iterations\n",
    "        for i in range(iterations):\n",
    "            self.model.train()\n",
    "\n",
    "            try:\n",
    "                x, seg_ids, y = next(loader)\n",
    "            except:  # after an loader running out\n",
    "                loader = iter(train_loader)\n",
    "                x, seg_ids, y = next(loader)\n",
    "            loss, _ = self.forward_pass(x, seg_ids, y)\n",
    "            \n",
    "#             loss.backward()\n",
    "            with amp.scale_loss(loss, self.optimizer) as scaled_loss:\n",
    "                scaled_loss.backward()\n",
    "\n",
    "            # gradient accumulation for larger batch size effect with smaller memory usage\n",
    "            if (i+1) % grad_accumulation_steps == 0:   # Wait for several backward steps\n",
    "                self.optimizer.step()                  # Now we can do an optimizer step\n",
    "                self.optimizer.zero_grad()\n",
    "                self.scheduler.step()\n",
    "\n",
    "            running_loss += loss.item()\n",
    "            \n",
    "            # verbose and checkpoint\n",
    "            if (i+1) % self.print_freq == 0 or (i+1) == iterations:\n",
    "                print(f'Iteration {i+1}:')\n",
    "                train_rho, _ = self.check_metric(train_loader, num_batches=50)\n",
    "                print('{\"metric\": \"Loss\", \"value\": %.4f}' % (running_loss/(i+1),))\n",
    "                print('{\"metric\": \"Rho\", \"value\": %.4f}' % (train_rho,))\n",
    "                \n",
    "                val_rho, val_loss, val_scores = self.check_metric(val_loader)\n",
    "                print('{\"metric\": \"Val. Loss\", \"value\": %.4f}' % (val_loss,))\n",
    "                print('{\"metric\": \"Val. Rho\", \"value\": %.4f}' % (val_rho,))\n",
    "                \n",
    "                self.loss_history.append(running_loss/(i+1))\n",
    "                self.val_loss_history.append(val_loss)\n",
    "                self.rho_history.append(train_rho)\n",
    "                self.val_rho_history.append(val_rho)\n",
    "                self.val_preds.append(val_scores)\n",
    "                \n",
    "                if val_loss < self.best_val_loss:\n",
    "                    print('updating best val loss...')\n",
    "                    self.best_val_loss = val_loss\n",
    "                if val_rho > self.best_val_rho:\n",
    "                    print('updating best val Spearman R...')\n",
    "                    self.best_val_rho = val_rho\n",
    "                    self.save_checkpoint(i+1)\n",
    "                \n",
    "                torch.cuda.empty_cache()\n",
    "                print()\n",
    "                \n",
    "            if (time.time() - start_time) > 29000:\n",
    "                break\n",
    "        \n",
    "        self.save_final_state()\n",
    "\n",
    "    def check_metric(self, loader, num_batches=None):\n",
    "        \"\"\"Calculate metrics for validation\n",
    "        \"\"\"\n",
    "        self.model.eval()\n",
    "        targets, scores, losses = [], [], []\n",
    "        with torch.no_grad():\n",
    "            for t, (x, seg_ids, y) in enumerate(loader):\n",
    "                l, score = self.forward_pass(x, seg_ids, y)\n",
    "                targets.append(y.cpu().numpy())\n",
    "                scores.append(score.cpu().numpy())\n",
    "                losses.append(l.item())\n",
    "                if num_batches is not None and (t+1) == num_batches:\n",
    "                    break\n",
    "\n",
    "        targets = np.concatenate(targets)\n",
    "        scores = np.concatenate(scores)\n",
    "        rho = compute_rho(targets, scores)\n",
    "        loss = np.mean(losses)\n",
    "        \n",
    "        if num_batches is None:\n",
    "            return rho, loss, scores\n",
    "        return rho, loss"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "_kg_hide-input": false
   },
   "outputs": [],
   "source": [
    "# lr scheduler\n",
    "\n",
    "class OneCycleScheduler(object):\n",
    "    # one-cycle scheduler\n",
    "    SCHEDULES = set(['cosine', 'linear', 'linear_cosine'])\n",
    "\n",
    "    def __init__(self, optimizer, iterations, sched_profile='cosine', max_lr=3e-3,\n",
    "                 moms=(.95, .85), div_factor=25, warmup=0.3, final_div=None):\n",
    "\n",
    "        self.optimizer = optimizer\n",
    "        assert sched_profile in self.SCHEDULES\n",
    "        self.sched_profile = sched_profile\n",
    "\n",
    "        if isinstance(max_lr, list) or isinstance(max_lr, tuple):\n",
    "            if len(max_lr) != len(optimizer.param_groups):\n",
    "                raise ValueError(\"expected {} max_lr, got {}\".format(\n",
    "                    len(optimizer.param_groups), len(max_lr)))\n",
    "            self.max_lrs = list(max_lr)\n",
    "            self.init_lrs = [lr/div_factor for lr in self.max_lrs]\n",
    "        else:\n",
    "            self.max_lrs = [max_lr] * len(optimizer.param_groups)\n",
    "            self.init_lrs = [max_lr/div_factor] * len(optimizer.param_groups)\n",
    "\n",
    "        self.final_div = final_div\n",
    "        if self.final_div is None: self.final_div = div_factor*1e4\n",
    "        self.final_lrs = [lr/self.final_div for lr in self.max_lrs]\n",
    "        self.moms = moms\n",
    "\n",
    "        self.total_iteration = iterations\n",
    "        self.up_iteration = int(self.total_iteration * warmup)\n",
    "        self.down_iteration = self.total_iteration - self.up_iteration\n",
    "\n",
    "        self.curr_iter = 0\n",
    "        self._assign_lr_mom(self.init_lrs, [moms[0]]*len(optimizer.param_groups))\n",
    "\n",
    "    def _assign_lr_mom(self, lrs, moms):\n",
    "        for param_group, lr, mom in zip(self.optimizer.param_groups, lrs, moms):\n",
    "            param_group['lr'] = lr\n",
    "            param_group['betas'] = (mom, 0.999)\n",
    "\n",
    "    def _annealing_cos(self, start, end, pct):\n",
    "        cos_out = np.cos(np.pi * pct) + 1\n",
    "        return end + (start-end)/2 * cos_out\n",
    "\n",
    "    def _annealing_linear(self, start, end, pct):\n",
    "        return start + pct * (end-start)\n",
    "    \n",
    "    def _annealing_function(self, curr_iter):\n",
    "        if self.sched_profile == 'cosine':\n",
    "            return self._annealing_cos\n",
    "        if self.sched_profile == 'linear':\n",
    "            return self._annealing_linear\n",
    "        if self.sched_profile == 'linear_cosine':\n",
    "            if curr_iter <= self.up_iteration:\n",
    "                return self._annealing_linear\n",
    "            else:\n",
    "                return self._annealing_cos\n",
    "    \n",
    "    def step(self):\n",
    "        self.curr_iter += 1\n",
    "        anneal = self._annealing_function(self.curr_iter)\n",
    "\n",
    "        if self.curr_iter <= self.up_iteration:\n",
    "            pct = self.curr_iter / self.up_iteration\n",
    "            curr_lrs = [anneal(min_lr, max_lr, pct) \\\n",
    "                            for min_lr, max_lr in zip(self.init_lrs, self.max_lrs)]\n",
    "            curr_moms = [anneal(self.moms[0], self.moms[1], pct) \\\n",
    "                            for _ in range(len(self.optimizer.param_groups))]\n",
    "        else:\n",
    "            pct = (self.curr_iter-self.up_iteration) / self.down_iteration\n",
    "            curr_lrs = [anneal(max_lr, final_lr, pct) \\\n",
    "                            for max_lr, final_lr in zip(self.max_lrs, self.final_lrs)]\n",
    "            curr_moms = [anneal(self.moms[1], self.moms[0], pct) \\\n",
    "                            for _ in range(len(self.optimizer.param_groups))]\n",
    "\n",
    "        self._assign_lr_mom(curr_lrs, curr_moms)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Train"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "def run_train_and_valid():\n",
    "    oof_preds = np.zeros_like(train_tars)\n",
    "    logs = []\n",
    "\n",
    "    for i, (trn_idx, val_idx) in enumerate(cv_indices):\n",
    "        print(f'Fold {i+1}:')\n",
    "\n",
    "        # prepare datasets\n",
    "        x_train, seg_train, y_train = convert_lines(train_df.iloc[trn_idx], input_cols,\n",
    "                                                    MAX_SEQUENCE_LENGTH, tokenizer,\n",
    "                                                    misc_trunc=True, target=train_tars[trn_idx],\n",
    "                                                    sample_weighting=True)\n",
    "        x_val, seg_val = convert_lines(train_df.iloc[val_idx], input_cols,\n",
    "                                       MAX_SEQUENCE_LENGTH, tokenizer)\n",
    "        y_val = train_tars[val_idx]\n",
    "        y_val = np.hstack([y_val, np.ones(len(y_val), dtype='float32')[:,None]])\n",
    "\n",
    "        train_loader = prepare_loader(x_train, seg_train, y_train, batch_size, split='train')\n",
    "        val_loader, val_original_indices = prepare_loader(x_val, seg_val, y_val, 16, split='valid')\n",
    "\n",
    "        # initialize model and solver\n",
    "        num_train_steps = int(epochs_for_sched * len(train_loader) / grad_accumulation_steps)\n",
    "        ft_lrs = [0.8*lr, lr]\n",
    "        model, optimizer = model_optimizer_init(ft_lrs)\n",
    "        scheduler = OneCycleScheduler(optimizer, num_train_steps, sched_profile='linear', max_lr=ft_lrs,\n",
    "                                      div_factor=40, warmup=warmup_proportion)\n",
    "        model = model.to(device)\n",
    "        criterion = TruncLoss().to(device)\n",
    "        model, optimizer = amp.initialize(model, optimizer, opt_level=\"O1\", verbosity=0)\n",
    "        solver = NetSolver(model, criterion, optimizer, scheduler, checkpoint_iter, f'fld_{i}_'+output_model_file)\n",
    "\n",
    "        # train\n",
    "        n_iter = num_train_steps * grad_accumulation_steps\n",
    "        print('Start training.')\n",
    "        t0 = time.time()\n",
    "        solver.train((train_loader, val_loader), n_iter, t0)\n",
    "        print(f'Training fold-{i+1} complete in {time.time()-t0} seconds.')\n",
    "\n",
    "        # record validation results\n",
    "        oof_preds[val_idx] += solver.val_preds[-1][val_original_indices]\n",
    "        History = namedtuple('History', 'history')\n",
    "        history = History(history={'loss': solver.loss_history, 'val_loss': solver.val_loss_history,\n",
    "                                   'rho': solver.rho_history, 'val_rho': solver.val_rho_history})\n",
    "        logs.append(history)\n",
    "\n",
    "        # clean cache\n",
    "        torch.cuda.empty_cache()\n",
    "        print(str(torch.cuda.memory_allocated(device)/1e6 ) + 'M')\n",
    "        print(str(torch.cuda.memory_cached(device)/1e6 ) + 'M')\n",
    "        print()\n",
    "\n",
    "    oof_rho = compute_rho(train_tars, oof_preds)\n",
    "    print('{\"metric\": \"OOF Val. Rho\", \"value\": %.4f}' % (oof_rho,))\n",
    "    \n",
    "    return logs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "_kg_hide-output": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fold 1:\n",
      "Building model and optimizer...\n",
      "Start training.\n",
      "Iteration 470:\n",
      "{\"metric\": \"Loss\", \"value\": 5.0344}\n",
      "{\"metric\": \"Rho\", \"value\": 0.3255}\n",
      "{\"metric\": \"Val. Loss\", \"value\": 11.9982}\n",
      "{\"metric\": \"Val. Rho\", \"value\": 0.2937}\n",
      "updating best val loss...\n",
      "updating best val Spearman R...\n",
      "\n",
      "Iteration 940:\n",
      "{\"metric\": \"Loss\", \"value\": 4.6805}\n",
      "{\"metric\": \"Rho\", \"value\": 0.3878}\n",
      "{\"metric\": \"Val. Loss\", \"value\": 11.4788}\n",
      "{\"metric\": \"Val. Rho\", \"value\": 0.3620}\n",
      "updating best val loss...\n",
      "updating best val Spearman R...\n",
      "\n",
      "Iteration 1410:\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/lib/python3.6/site-packages/numpy/lib/function_base.py:2530: RuntimeWarning: invalid value encountered in true_divide\n",
      "  c /= stddev[:, None]\n",
      "/opt/conda/lib/python3.6/site-packages/numpy/lib/function_base.py:2531: RuntimeWarning: invalid value encountered in true_divide\n",
      "  c /= stddev[None, :]\n",
      "/opt/conda/lib/python3.6/site-packages/scipy/stats/_distn_infrastructure.py:877: RuntimeWarning: invalid value encountered in greater\n",
      "  return (self.a < x) & (x < self.b)\n",
      "/opt/conda/lib/python3.6/site-packages/scipy/stats/_distn_infrastructure.py:877: RuntimeWarning: invalid value encountered in less\n",
      "  return (self.a < x) & (x < self.b)\n",
      "/opt/conda/lib/python3.6/site-packages/scipy/stats/_distn_infrastructure.py:1831: RuntimeWarning: invalid value encountered in less_equal\n",
      "  cond2 = cond0 & (x <= self.a)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{\"metric\": \"Loss\", \"value\": 4.5060}\n",
      "{\"metric\": \"Rho\", \"value\": nan}\n",
      "{\"metric\": \"Val. Loss\", \"value\": 11.2519}\n",
      "{\"metric\": \"Val. Rho\", \"value\": 0.3756}\n",
      "updating best val loss...\n",
      "updating best val Spearman R...\n",
      "\n",
      "Iteration 1880:\n",
      "{\"metric\": \"Loss\", \"value\": 4.3933}\n",
      "{\"metric\": \"Rho\", \"value\": nan}\n",
      "{\"metric\": \"Val. Loss\", \"value\": 11.2878}\n",
      "{\"metric\": \"Val. Rho\", \"value\": 0.3823}\n",
      "updating best val Spearman R...\n",
      "\n",
      "Iteration 2350:\n",
      "{\"metric\": \"Loss\", \"value\": 4.2890}\n",
      "{\"metric\": \"Rho\", \"value\": 0.4974}\n",
      "{\"metric\": \"Val. Loss\", \"value\": 11.2489}\n",
      "{\"metric\": \"Val. Rho\", \"value\": 0.3854}\n",
      "updating best val loss...\n",
      "updating best val Spearman R...\n",
      "\n",
      "Iteration 2804:\n",
      "{\"metric\": \"Loss\", \"value\": 4.2139}\n",
      "{\"metric\": \"Rho\", \"value\": nan}\n",
      "{\"metric\": \"Val. Loss\", \"value\": 11.2376}\n",
      "{\"metric\": \"Val. Rho\", \"value\": 0.3895}\n",
      "updating best val loss...\n",
      "updating best val Spearman R...\n",
      "\n",
      "Training fold-1 complete in 1161.8809950351715 seconds.\n",
      "1989.543936M\n",
      "3160.408064M\n",
      "\n",
      "Fold 2:\n",
      "Building model and optimizer...\n",
      "Start training.\n",
      "Iteration 470:\n",
      "{\"metric\": \"Loss\", \"value\": 5.0720}\n",
      "{\"metric\": \"Rho\", \"value\": nan}\n",
      "{\"metric\": \"Val. Loss\", \"value\": 12.2012}\n",
      "{\"metric\": \"Val. Rho\", \"value\": 0.2589}\n",
      "updating best val loss...\n",
      "updating best val Spearman R...\n",
      "\n",
      "Iteration 940:\n",
      "{\"metric\": \"Loss\", \"value\": 4.7420}\n",
      "{\"metric\": \"Rho\", \"value\": 0.3925}\n",
      "{\"metric\": \"Val. Loss\", \"value\": 11.4594}\n",
      "{\"metric\": \"Val. Rho\", \"value\": 0.3415}\n",
      "updating best val loss...\n",
      "updating best val Spearman R...\n",
      "\n",
      "Iteration 1410:\n",
      "{\"metric\": \"Loss\", \"value\": 4.5448}\n",
      "{\"metric\": \"Rho\", \"value\": 0.4360}\n",
      "{\"metric\": \"Val. Loss\", \"value\": 11.2945}\n",
      "{\"metric\": \"Val. Rho\", \"value\": 0.3560}\n",
      "updating best val loss...\n",
      "updating best val Spearman R...\n",
      "\n",
      "Iteration 1880:\n",
      "{\"metric\": \"Loss\", \"value\": 4.4288}\n",
      "{\"metric\": \"Rho\", \"value\": 0.4795}\n",
      "{\"metric\": \"Val. Loss\", \"value\": 11.1889}\n",
      "{\"metric\": \"Val. Rho\", \"value\": 0.3701}\n",
      "updating best val loss...\n",
      "updating best val Spearman R...\n",
      "\n",
      "Iteration 2350:\n",
      "{\"metric\": \"Loss\", \"value\": 4.3172}\n",
      "{\"metric\": \"Rho\", \"value\": nan}\n",
      "{\"metric\": \"Val. Loss\", \"value\": 11.2574}\n",
      "{\"metric\": \"Val. Rho\", \"value\": 0.3733}\n",
      "updating best val Spearman R...\n",
      "\n",
      "Iteration 2790:\n",
      "{\"metric\": \"Loss\", \"value\": 4.2471}\n",
      "{\"metric\": \"Rho\", \"value\": 0.5258}\n",
      "{\"metric\": \"Val. Loss\", \"value\": 11.2216}\n",
      "{\"metric\": \"Val. Rho\", \"value\": 0.3760}\n",
      "updating best val Spearman R...\n",
      "\n",
      "Training fold-2 complete in 1162.4671773910522 seconds.\n",
      "2883.91424M\n",
      "4284.481536M\n",
      "\n",
      "Fold 3:\n",
      "Building model and optimizer...\n",
      "Start training.\n",
      "Iteration 470:\n",
      "{\"metric\": \"Loss\", \"value\": 5.1351}\n",
      "{\"metric\": \"Rho\", \"value\": 0.2895}\n",
      "{\"metric\": \"Val. Loss\", \"value\": 12.2206}\n",
      "{\"metric\": \"Val. Rho\", \"value\": 0.2760}\n",
      "updating best val loss...\n",
      "updating best val Spearman R...\n",
      "\n",
      "Iteration 940:\n",
      "{\"metric\": \"Loss\", \"value\": 4.7552}\n",
      "{\"metric\": \"Rho\", \"value\": 0.3762}\n",
      "{\"metric\": \"Val. Loss\", \"value\": 11.3020}\n",
      "{\"metric\": \"Val. Rho\", \"value\": 0.3549}\n",
      "updating best val loss...\n",
      "updating best val Spearman R...\n",
      "\n",
      "Iteration 1410:\n",
      "{\"metric\": \"Loss\", \"value\": 4.5557}\n",
      "{\"metric\": \"Rho\", \"value\": 0.4358}\n",
      "{\"metric\": \"Val. Loss\", \"value\": 11.2359}\n",
      "{\"metric\": \"Val. Rho\", \"value\": 0.3803}\n",
      "updating best val loss...\n",
      "updating best val Spearman R...\n",
      "\n",
      "Iteration 1880:\n",
      "{\"metric\": \"Loss\", \"value\": 4.4367}\n",
      "{\"metric\": \"Rho\", \"value\": nan}\n",
      "{\"metric\": \"Val. Loss\", \"value\": 11.0250}\n",
      "{\"metric\": \"Val. Rho\", \"value\": 0.3971}\n",
      "updating best val loss...\n",
      "updating best val Spearman R...\n",
      "\n",
      "Iteration 2350:\n",
      "{\"metric\": \"Loss\", \"value\": 4.3283}\n",
      "{\"metric\": \"Rho\", \"value\": nan}\n",
      "{\"metric\": \"Val. Loss\", \"value\": 11.0474}\n",
      "{\"metric\": \"Val. Rho\", \"value\": 0.3985}\n",
      "updating best val Spearman R...\n",
      "\n",
      "Iteration 2820:\n",
      "{\"metric\": \"Loss\", \"value\": 4.2480}\n",
      "{\"metric\": \"Rho\", \"value\": nan}\n",
      "{\"metric\": \"Val. Loss\", \"value\": 10.9993}\n",
      "{\"metric\": \"Val. Rho\", \"value\": 0.4012}\n",
      "updating best val loss...\n",
      "updating best val Spearman R...\n",
      "\n",
      "Iteration 2826:\n",
      "{\"metric\": \"Loss\", \"value\": 4.2468}\n",
      "{\"metric\": \"Rho\", \"value\": nan}\n",
      "{\"metric\": \"Val. Loss\", \"value\": 10.9993}\n",
      "{\"metric\": \"Val. Rho\", \"value\": 0.4012}\n",
      "\n",
      "Training fold-3 complete in 1210.9680166244507 seconds.\n",
      "3803.984384M\n",
      "6136.266752M\n",
      "\n",
      "Fold 4:\n",
      "Building model and optimizer...\n",
      "Start training.\n",
      "Iteration 470:\n",
      "{\"metric\": \"Loss\", \"value\": 5.0133}\n",
      "{\"metric\": \"Rho\", \"value\": nan}\n",
      "{\"metric\": \"Val. Loss\", \"value\": 11.8365}\n",
      "{\"metric\": \"Val. Rho\", \"value\": 0.2889}\n",
      "updating best val loss...\n",
      "updating best val Spearman R...\n",
      "\n",
      "Iteration 940:\n",
      "{\"metric\": \"Loss\", \"value\": 4.6909}\n",
      "{\"metric\": \"Rho\", \"value\": 0.4105}\n",
      "{\"metric\": \"Val. Loss\", \"value\": 11.2148}\n",
      "{\"metric\": \"Val. Rho\", \"value\": 0.3503}\n",
      "updating best val loss...\n",
      "updating best val Spearman R...\n",
      "\n",
      "Iteration 1410:\n",
      "{\"metric\": \"Loss\", \"value\": 4.5198}\n",
      "{\"metric\": \"Rho\", \"value\": nan}\n",
      "{\"metric\": \"Val. Loss\", \"value\": 11.1039}\n",
      "{\"metric\": \"Val. Rho\", \"value\": 0.3743}\n",
      "updating best val loss...\n",
      "updating best val Spearman R...\n",
      "\n",
      "Iteration 1880:\n",
      "{\"metric\": \"Loss\", \"value\": 4.4047}\n",
      "{\"metric\": \"Rho\", \"value\": 0.4760}\n",
      "{\"metric\": \"Val. Loss\", \"value\": 10.9587}\n",
      "{\"metric\": \"Val. Rho\", \"value\": 0.3875}\n",
      "updating best val loss...\n",
      "updating best val Spearman R...\n",
      "\n",
      "Iteration 2350:\n",
      "{\"metric\": \"Loss\", \"value\": 4.3033}\n",
      "{\"metric\": \"Rho\", \"value\": nan}\n",
      "{\"metric\": \"Val. Loss\", \"value\": 11.0427}\n",
      "{\"metric\": \"Val. Rho\", \"value\": 0.3864}\n",
      "\n",
      "Iteration 2810:\n",
      "{\"metric\": \"Loss\", \"value\": 4.2268}\n",
      "{\"metric\": \"Rho\", \"value\": 0.5375}\n",
      "{\"metric\": \"Val. Loss\", \"value\": 11.0230}\n",
      "{\"metric\": \"Val. Rho\", \"value\": 0.3859}\n",
      "\n",
      "Training fold-4 complete in 1188.774957895279 seconds.\n",
      "4726.774272M\n",
      "7031.750656M\n",
      "\n",
      "Fold 5:\n",
      "Building model and optimizer...\n",
      "Start training.\n",
      "Iteration 470:\n",
      "{\"metric\": \"Loss\", \"value\": 5.1056}\n",
      "{\"metric\": \"Rho\", \"value\": nan}\n",
      "{\"metric\": \"Val. Loss\", \"value\": 12.0159}\n",
      "{\"metric\": \"Val. Rho\", \"value\": 0.2659}\n",
      "updating best val loss...\n",
      "updating best val Spearman R...\n",
      "\n",
      "Iteration 940:\n",
      "{\"metric\": \"Loss\", \"value\": 4.7645}\n",
      "{\"metric\": \"Rho\", \"value\": nan}\n",
      "{\"metric\": \"Val. Loss\", \"value\": 11.5041}\n",
      "{\"metric\": \"Val. Rho\", \"value\": 0.3393}\n",
      "updating best val loss...\n",
      "updating best val Spearman R...\n",
      "\n",
      "Iteration 1410:\n",
      "{\"metric\": \"Loss\", \"value\": 4.5688}\n",
      "{\"metric\": \"Rho\", \"value\": 0.3993}\n",
      "{\"metric\": \"Val. Loss\", \"value\": 11.1708}\n",
      "{\"metric\": \"Val. Rho\", \"value\": 0.3642}\n",
      "updating best val loss...\n",
      "updating best val Spearman R...\n",
      "\n",
      "Iteration 1880:\n",
      "{\"metric\": \"Loss\", \"value\": 4.4538}\n",
      "{\"metric\": \"Rho\", \"value\": 0.4756}\n",
      "{\"metric\": \"Val. Loss\", \"value\": 11.1465}\n",
      "{\"metric\": \"Val. Rho\", \"value\": 0.3786}\n",
      "updating best val loss...\n",
      "updating best val Spearman R...\n",
      "\n",
      "Iteration 2350:\n",
      "{\"metric\": \"Loss\", \"value\": 4.3453}\n",
      "{\"metric\": \"Rho\", \"value\": nan}\n",
      "{\"metric\": \"Val. Loss\", \"value\": 11.1475}\n",
      "{\"metric\": \"Val. Rho\", \"value\": 0.3768}\n",
      "\n",
      "Iteration 2804:\n",
      "{\"metric\": \"Loss\", \"value\": 4.2684}\n",
      "{\"metric\": \"Rho\", \"value\": 0.5005}\n",
      "{\"metric\": \"Val. Loss\", \"value\": 11.1441}\n",
      "{\"metric\": \"Val. Rho\", \"value\": 0.3796}\n",
      "updating best val loss...\n",
      "updating best val Spearman R...\n",
      "\n",
      "Training fold-5 complete in 1192.4989564418793 seconds.\n",
      "5626.345984M\n",
      "7956.594688M\n",
      "\n",
      "{\"metric\": \"OOF Val. Rho\", \"value\": 0.3857}\n"
     ]
    }
   ],
   "source": [
    "logs = run_train_and_valid()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Visualization"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "_kg_hide-input": true
   },
   "outputs": [],
   "source": [
    "import plotly.express as px\n",
    "import plotly.graph_objects as go\n",
    "from plotly.subplots import make_subplots"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "_kg_hide-input": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "        <script type=\"text/javascript\">\n",
       "        window.PlotlyConfig = {MathJaxConfig: 'local'};\n",
       "        if (window.MathJax) {MathJax.Hub.Config({SVG: {font: \"STIX-Web\"}});}\n",
       "        if (typeof require !== 'undefined') {\n",
       "        require.undef(\"plotly\");\n",
       "        requirejs.config({\n",
       "            paths: {\n",
       "                'plotly': ['https://cdn.plot.ly/plotly-latest.min']\n",
       "            }\n",
       "        });\n",
       "        require(['plotly'], function(Plotly) {\n",
       "            window._Plotly = Plotly;\n",
       "        });\n",
       "        }\n",
       "        </script>\n",
       "        "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "        \n",
       "        \n",
       "            <div id=\"b0eddee6-a772-4669-bdd0-be7615224409\" class=\"plotly-graph-div\" style=\"height:525px; width:100%;\"></div>\n",
       "            <script type=\"text/javascript\">\n",
       "                require([\"plotly\"], function(Plotly) {\n",
       "                    window.PLOTLYENV=window.PLOTLYENV || {};\n",
       "                    \n",
       "                if (document.getElementById(\"b0eddee6-a772-4669-bdd0-be7615224409\")) {\n",
       "                    Plotly.newPlot(\n",
       "                        'b0eddee6-a772-4669-bdd0-be7615224409',\n",
       "                        [{\"line\": {\"color\": \"#d32f2f\", \"width\": 2}, \"mode\": \"lines\", \"name\": \"Train loss - Fold #0\", \"type\": \"scatter\", \"visible\": true, \"x\": [0, 1, 2], \"xaxis\": \"x\", \"y\": [5.034446132436711, 4.680456539925109, 4.5060292698812825, 4.39325359045191, 4.289046794404375, 4.2138607573407185], \"yaxis\": \"y\"}, {\"line\": {\"color\": \"#ef5350\", \"width\": 2}, \"mode\": \"lines+markers\", \"name\": \"Valid loss - Fold #0\", \"type\": \"scatter\", \"visible\": true, \"x\": [0, 1, 2], \"xaxis\": \"x\", \"y\": [11.99823576525638, 11.478814727381655, 11.251938393241481, 11.287782229875264, 11.248868578358701, 11.237603112270957], \"yaxis\": \"y\"}, {\"line\": {\"color\": \"#ef5350\", \"width\": 2}, \"mode\": \"lines+markers\", \"name\": \"Valid RHO - Fold #0\", \"type\": \"scatter\", \"visible\": true, \"x\": [0, 1, 2], \"xaxis\": \"x2\", \"y\": [0.2936897837815663, 0.3620294826900377, 0.37556813040678194, 0.38230126415667, 0.3854270728325464, 0.3895177628389654], \"yaxis\": \"y2\"}, {\"line\": {\"color\": \"#303f9f\", \"width\": 2}, \"mode\": \"lines\", \"name\": \"Train loss - Fold #1\", \"type\": \"scatter\", \"visible\": \"legendonly\", \"x\": [0, 1, 2], \"xaxis\": \"x\", \"y\": [5.071952038622917, 4.741975001324999, 4.544825900869166, 4.42877178953049, 4.317210542496214, 4.247102875179714], \"yaxis\": \"y\"}, {\"line\": {\"color\": \"#5c6bc0\", \"width\": 2}, \"mode\": \"lines+markers\", \"name\": \"Valid loss - Fold #1\", \"type\": \"scatter\", \"visible\": \"legendonly\", \"x\": [0, 1, 2], \"xaxis\": \"x\", \"y\": [12.201167395240383, 11.459427293978239, 11.29453218610663, 11.188899278640747, 11.257444820905986, 11.221625428450736], \"yaxis\": \"y\"}, {\"line\": {\"color\": \"#5c6bc0\", \"width\": 2}, \"mode\": \"lines+markers\", \"name\": \"Valid RHO - Fold #1\", \"type\": \"scatter\", \"visible\": \"legendonly\", \"x\": [0, 1, 2], \"xaxis\": \"x2\", \"y\": [0.2588876026101264, 0.34152244151913785, 0.35599695482065913, 0.3700512432581659, 0.3733169927251788, 0.37599887834989265], \"yaxis\": \"y2\"}, {\"line\": {\"color\": \"#00796b\", \"width\": 2}, \"mode\": \"lines\", \"name\": \"Train loss - Fold #2\", \"type\": \"scatter\", \"visible\": \"legendonly\", \"x\": [0, 1, 2], \"xaxis\": \"x\", \"y\": [5.135143093859896, 4.755226990009876, 4.555671496256023, 4.436676241108712, 4.32827054957126, 4.247962122014228, 4.2467879513351106], \"yaxis\": \"y\"}, {\"line\": {\"color\": \"#26a69a\", \"width\": 2}, \"mode\": \"lines+markers\", \"name\": \"Valid loss - Fold #2\", \"type\": \"scatter\", \"visible\": \"legendonly\", \"x\": [0, 1, 2], \"xaxis\": \"x\", \"y\": [12.22059739263434, 11.301975425920988, 11.235854450025057, 11.025032470100804, 11.047362064060412, 10.99926105298494, 10.999268694927817], \"yaxis\": \"y\"}, {\"line\": {\"color\": \"#26a69a\", \"width\": 2}, \"mode\": \"lines+markers\", \"name\": \"Valid RHO - Fold #2\", \"type\": \"scatter\", \"visible\": \"legendonly\", \"x\": [0, 1, 2], \"xaxis\": \"x2\", \"y\": [0.2760393939337281, 0.3549366637801051, 0.3802806937382235, 0.39712654650784446, 0.3985498261927776, 0.40122544571037894, 0.4012134887052377], \"yaxis\": \"y2\"}, {\"line\": {\"color\": \"#fbc02d\", \"width\": 2}, \"mode\": \"lines\", \"name\": \"Train loss - Fold #3\", \"type\": \"scatter\", \"visible\": \"legendonly\", \"x\": [0, 1, 2], \"xaxis\": \"x\", \"y\": [5.013322617652569, 4.690871597350912, 4.519842135652583, 4.404684971875333, 4.303281039582922, 4.226783520556006], \"yaxis\": \"y\"}, {\"line\": {\"color\": \"#ffeb3b\", \"width\": 2}, \"mode\": \"lines+markers\", \"name\": \"Valid loss - Fold #3\", \"type\": \"scatter\", \"visible\": \"legendonly\", \"x\": [0, 1, 2], \"xaxis\": \"x\", \"y\": [11.836458733207301, 11.214763453132228, 11.103880079169022, 10.958698925219084, 11.042734547665244, 11.02301365450809], \"yaxis\": \"y\"}, {\"line\": {\"color\": \"#ffeb3b\", \"width\": 2}, \"mode\": \"lines+markers\", \"name\": \"Valid RHO - Fold #3\", \"type\": \"scatter\", \"visible\": \"legendonly\", \"x\": [0, 1, 2], \"xaxis\": \"x2\", \"y\": [0.28886066522332676, 0.3503465021637133, 0.3742901202666849, 0.38750688733834476, 0.3863926751252596, 0.3859499682752289], \"yaxis\": \"y2\"}, {\"line\": {\"color\": \"#5d4037\", \"width\": 2}, \"mode\": \"lines\", \"name\": \"Train loss - Fold #4\", \"type\": \"scatter\", \"visible\": \"legendonly\", \"x\": [0, 1, 2], \"xaxis\": \"x\", \"y\": [5.105632617625784, 4.764531157625482, 4.568785853927017, 4.453755878133976, 4.345286614032502, 4.268396342191819], \"yaxis\": \"y\"}, {\"line\": {\"color\": \"#8d6e63\", \"width\": 2}, \"mode\": \"lines+markers\", \"name\": \"Valid loss - Fold #4\", \"type\": \"scatter\", \"visible\": \"legendonly\", \"x\": [0, 1, 2], \"xaxis\": \"x\", \"y\": [12.01588171406796, 11.504147253538433, 11.17078833830984, 11.146546263443796, 11.147525686966745, 11.144143706873843], \"yaxis\": \"y\"}, {\"line\": {\"color\": \"#8d6e63\", \"width\": 2}, \"mode\": \"lines+markers\", \"name\": \"Valid RHO - Fold #4\", \"type\": \"scatter\", \"visible\": \"legendonly\", \"x\": [0, 1, 2], \"xaxis\": \"x2\", \"y\": [0.26594876834764847, 0.33932905387880363, 0.36421568746884536, 0.3785705114587929, 0.37682378650264975, 0.3796349039847778], \"yaxis\": \"y2\"}],\n",
       "                        {\"annotations\": [{\"font\": {\"size\": 14}, \"showarrow\": false, \"text\": \"Train / valid losses\", \"x\": 0.5, \"xanchor\": \"center\", \"xref\": \"paper\", \"y\": 1.0, \"yanchor\": \"bottom\", \"yref\": \"paper\"}, {\"font\": {\"size\": 14}, \"showarrow\": false, \"text\": \"Train / valid AUC\", \"x\": 0.5, \"xanchor\": \"center\", \"xref\": \"paper\", \"y\": 0.5, \"yanchor\": \"bottom\", \"yref\": \"paper\"}], \"template\": {\"data\": {\"bar\": [{\"error_x\": {\"color\": \"#2a3f5f\"}, \"error_y\": {\"color\": \"#2a3f5f\"}, \"marker\": {\"line\": {\"color\": \"#E5ECF6\", \"width\": 0.5}}, \"type\": \"bar\"}], \"barpolar\": [{\"marker\": {\"line\": {\"color\": \"#E5ECF6\", \"width\": 0.5}}, \"type\": \"barpolar\"}], \"carpet\": [{\"aaxis\": {\"endlinecolor\": \"#2a3f5f\", \"gridcolor\": \"white\", \"linecolor\": \"white\", \"minorgridcolor\": \"white\", \"startlinecolor\": \"#2a3f5f\"}, \"baxis\": {\"endlinecolor\": \"#2a3f5f\", \"gridcolor\": \"white\", \"linecolor\": \"white\", \"minorgridcolor\": \"white\", \"startlinecolor\": \"#2a3f5f\"}, \"type\": \"carpet\"}], \"choropleth\": [{\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}, \"type\": \"choropleth\"}], \"contour\": [{\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}, \"colorscale\": [[0.0, \"#0d0887\"], [0.1111111111111111, \"#46039f\"], [0.2222222222222222, \"#7201a8\"], [0.3333333333333333, \"#9c179e\"], [0.4444444444444444, \"#bd3786\"], [0.5555555555555556, \"#d8576b\"], [0.6666666666666666, \"#ed7953\"], [0.7777777777777778, \"#fb9f3a\"], [0.8888888888888888, \"#fdca26\"], [1.0, \"#f0f921\"]], \"type\": \"contour\"}], \"contourcarpet\": [{\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}, \"type\": \"contourcarpet\"}], \"heatmap\": [{\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}, \"colorscale\": [[0.0, \"#0d0887\"], [0.1111111111111111, \"#46039f\"], [0.2222222222222222, \"#7201a8\"], [0.3333333333333333, \"#9c179e\"], [0.4444444444444444, \"#bd3786\"], [0.5555555555555556, \"#d8576b\"], [0.6666666666666666, \"#ed7953\"], [0.7777777777777778, \"#fb9f3a\"], [0.8888888888888888, \"#fdca26\"], [1.0, \"#f0f921\"]], \"type\": \"heatmap\"}], \"heatmapgl\": [{\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}, \"colorscale\": [[0.0, \"#0d0887\"], [0.1111111111111111, \"#46039f\"], [0.2222222222222222, \"#7201a8\"], [0.3333333333333333, \"#9c179e\"], [0.4444444444444444, \"#bd3786\"], [0.5555555555555556, \"#d8576b\"], [0.6666666666666666, \"#ed7953\"], [0.7777777777777778, \"#fb9f3a\"], [0.8888888888888888, \"#fdca26\"], [1.0, \"#f0f921\"]], \"type\": \"heatmapgl\"}], \"histogram\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"histogram\"}], \"histogram2d\": [{\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}, \"colorscale\": [[0.0, \"#0d0887\"], [0.1111111111111111, \"#46039f\"], [0.2222222222222222, \"#7201a8\"], [0.3333333333333333, \"#9c179e\"], [0.4444444444444444, \"#bd3786\"], [0.5555555555555556, \"#d8576b\"], [0.6666666666666666, \"#ed7953\"], [0.7777777777777778, \"#fb9f3a\"], [0.8888888888888888, \"#fdca26\"], [1.0, \"#f0f921\"]], \"type\": \"histogram2d\"}], \"histogram2dcontour\": [{\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}, \"colorscale\": [[0.0, \"#0d0887\"], [0.1111111111111111, \"#46039f\"], [0.2222222222222222, \"#7201a8\"], [0.3333333333333333, \"#9c179e\"], [0.4444444444444444, \"#bd3786\"], [0.5555555555555556, \"#d8576b\"], [0.6666666666666666, \"#ed7953\"], [0.7777777777777778, \"#fb9f3a\"], [0.8888888888888888, \"#fdca26\"], [1.0, \"#f0f921\"]], \"type\": \"histogram2dcontour\"}], \"mesh3d\": [{\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}, \"type\": \"mesh3d\"}], \"parcoords\": [{\"line\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"parcoords\"}], \"scatter\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"scatter\"}], \"scatter3d\": [{\"line\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"scatter3d\"}], \"scattercarpet\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"scattercarpet\"}], \"scattergeo\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"scattergeo\"}], \"scattergl\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"scattergl\"}], \"scattermapbox\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"scattermapbox\"}], \"scatterpolar\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"scatterpolar\"}], \"scatterpolargl\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"scatterpolargl\"}], \"scatterternary\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"scatterternary\"}], \"surface\": [{\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}, \"colorscale\": [[0.0, \"#0d0887\"], [0.1111111111111111, \"#46039f\"], [0.2222222222222222, \"#7201a8\"], [0.3333333333333333, \"#9c179e\"], [0.4444444444444444, \"#bd3786\"], [0.5555555555555556, \"#d8576b\"], [0.6666666666666666, \"#ed7953\"], [0.7777777777777778, \"#fb9f3a\"], [0.8888888888888888, \"#fdca26\"], [1.0, \"#f0f921\"]], \"type\": \"surface\"}], \"table\": [{\"cells\": {\"fill\": {\"color\": \"#EBF0F8\"}, \"line\": {\"color\": \"white\"}}, \"header\": {\"fill\": {\"color\": \"#C8D4E3\"}, \"line\": {\"color\": \"white\"}}, \"type\": \"table\"}]}, \"layout\": {\"annotationdefaults\": {\"arrowcolor\": \"#2a3f5f\", \"arrowhead\": 0, \"arrowwidth\": 1}, \"colorscale\": {\"diverging\": [[0, \"#8e0152\"], [0.1, \"#c51b7d\"], [0.2, \"#de77ae\"], [0.3, \"#f1b6da\"], [0.4, \"#fde0ef\"], [0.5, \"#f7f7f7\"], [0.6, \"#e6f5d0\"], [0.7, \"#b8e186\"], [0.8, \"#7fbc41\"], [0.9, \"#4d9221\"], [1, \"#276419\"]], \"sequential\": [[0.0, \"#0d0887\"], [0.1111111111111111, \"#46039f\"], [0.2222222222222222, \"#7201a8\"], [0.3333333333333333, \"#9c179e\"], [0.4444444444444444, \"#bd3786\"], [0.5555555555555556, \"#d8576b\"], [0.6666666666666666, \"#ed7953\"], [0.7777777777777778, \"#fb9f3a\"], [0.8888888888888888, \"#fdca26\"], [1.0, \"#f0f921\"]], \"sequentialminus\": [[0.0, \"#0d0887\"], [0.1111111111111111, \"#46039f\"], [0.2222222222222222, \"#7201a8\"], [0.3333333333333333, \"#9c179e\"], [0.4444444444444444, \"#bd3786\"], [0.5555555555555556, \"#d8576b\"], [0.6666666666666666, \"#ed7953\"], [0.7777777777777778, \"#fb9f3a\"], [0.8888888888888888, \"#fdca26\"], [1.0, \"#f0f921\"]]}, \"colorway\": [\"#636efa\", \"#EF553B\", \"#00cc96\", \"#ab63fa\", \"#FFA15A\", \"#19d3f3\", \"#FF6692\", \"#B6E880\", \"#FF97FF\", \"#FECB52\"], \"font\": {\"color\": \"#2a3f5f\"}, \"geo\": {\"bgcolor\": \"white\", \"lakecolor\": \"white\", \"landcolor\": \"#E5ECF6\", \"showlakes\": true, \"showland\": true, \"subunitcolor\": \"white\"}, \"hoverlabel\": {\"align\": \"left\"}, \"hovermode\": \"closest\", \"mapbox\": {\"style\": \"light\"}, \"paper_bgcolor\": \"white\", \"plot_bgcolor\": \"#E5ECF6\", \"polar\": {\"angularaxis\": {\"gridcolor\": \"white\", \"linecolor\": \"white\", \"ticks\": \"\"}, \"bgcolor\": \"#E5ECF6\", \"radialaxis\": {\"gridcolor\": \"white\", \"linecolor\": \"white\", \"ticks\": \"\"}}, \"scene\": {\"xaxis\": {\"backgroundcolor\": \"#E5ECF6\", \"gridcolor\": \"white\", \"gridwidth\": 2, \"linecolor\": \"white\", \"showbackground\": true, \"ticks\": \"\", \"zerolinecolor\": \"white\"}, \"yaxis\": {\"backgroundcolor\": \"#E5ECF6\", \"gridcolor\": \"white\", \"gridwidth\": 2, \"linecolor\": \"white\", \"showbackground\": true, \"ticks\": \"\", \"zerolinecolor\": \"white\"}, \"zaxis\": {\"backgroundcolor\": \"#E5ECF6\", \"gridcolor\": \"white\", \"gridwidth\": 2, \"linecolor\": \"white\", \"showbackground\": true, \"ticks\": \"\", \"zerolinecolor\": \"white\"}}, \"shapedefaults\": {\"line\": {\"color\": \"#2a3f5f\"}}, \"ternary\": {\"aaxis\": {\"gridcolor\": \"white\", \"linecolor\": \"white\", \"ticks\": \"\"}, \"baxis\": {\"gridcolor\": \"white\", \"linecolor\": \"white\", \"ticks\": \"\"}, \"bgcolor\": \"#E5ECF6\", \"caxis\": {\"gridcolor\": \"white\", \"linecolor\": \"white\", \"ticks\": \"\"}}, \"title\": {\"x\": 0.05}, \"xaxis\": {\"automargin\": true, \"gridcolor\": \"white\", \"linecolor\": \"white\", \"ticks\": \"\", \"zerolinecolor\": \"white\", \"zerolinewidth\": 2}, \"yaxis\": {\"automargin\": true, \"gridcolor\": \"white\", \"linecolor\": \"white\", \"ticks\": \"\", \"zerolinecolor\": \"white\", \"zerolinewidth\": 2}}}, \"xaxis\": {\"anchor\": \"y\", \"domain\": [0.0, 1.0]}, \"xaxis2\": {\"anchor\": \"y2\", \"domain\": [0.0, 1.0]}, \"yaxis\": {\"anchor\": \"x\", \"domain\": [0.575, 1.0]}, \"yaxis2\": {\"anchor\": \"x2\", \"domain\": [0.0, 0.425]}},\n",
       "                        {\"responsive\": true}\n",
       "                    ).then(function(){\n",
       "                            \n",
       "var gd = document.getElementById('b0eddee6-a772-4669-bdd0-be7615224409');\n",
       "var x = new MutationObserver(function (mutations, observer) {{\n",
       "        var display = window.getComputedStyle(gd).display;\n",
       "        if (!display || display === 'none') {{\n",
       "            console.log([gd, 'removed!']);\n",
       "            Plotly.purge(gd);\n",
       "            observer.disconnect();\n",
       "        }}\n",
       "}});\n",
       "\n",
       "// Listen for the removal of the full notebook cells\n",
       "var notebookContainer = gd.closest('#notebook-container');\n",
       "if (notebookContainer) {{\n",
       "    x.observe(notebookContainer, {childList: true});\n",
       "}}\n",
       "\n",
       "// Listen for the clearing of the current output cell\n",
       "var outputEl = gd.closest('.output');\n",
       "if (outputEl) {{\n",
       "    x.observe(outputEl, {childList: true});\n",
       "}}\n",
       "\n",
       "                        })\n",
       "                };\n",
       "                });\n",
       "            </script>\n",
       "        </div>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "fig = make_subplots(rows=2, cols=1)\n",
    "\n",
    "colors = [\n",
    "    ('#d32f2f', '#ef5350'),\n",
    "    ('#303f9f', '#5c6bc0'),\n",
    "    ('#00796b', '#26a69a'),\n",
    "    ('#fbc02d', '#ffeb3b'),\n",
    "    ('#5d4037', '#8d6e63'),\n",
    "]\n",
    "\n",
    "for fold, history in enumerate(logs):\n",
    "    \n",
    "    fig.add_trace(go.Scatter(x=np.arange(epochs_for_sched),\n",
    "                             y=history.history['loss'],\n",
    "                             mode='lines',\n",
    "                             visible='legendonly' if fold > 0 else True,\n",
    "                             line=dict(color=colors[fold][0], width=2),\n",
    "                             name='Train loss - Fold #{}'.format(fold)),\n",
    "                 row=1, col=1)\n",
    "\n",
    "    fig.add_trace(go.Scatter(x=np.arange(epochs_for_sched),\n",
    "                             y=history.history['val_loss'],\n",
    "                             mode='lines+markers',\n",
    "                             visible='legendonly' if fold > 0 else True,\n",
    "                             line=dict(color=colors[fold][1], width=2),\n",
    "                             name='Valid loss - Fold #{}'.format(fold)),\n",
    "                 row=1, col=1)\n",
    "    \n",
    "    fig.add_trace(go.Scatter(x=np.arange(epochs_for_sched),\n",
    "                             y=history.history['val_rho'],\n",
    "                             mode='lines+markers',\n",
    "                             visible='legendonly' if fold > 0 else True,\n",
    "                             line=dict(color=colors[fold][1], width=2),\n",
    "                             name='Valid RHO - Fold #{}'.format(fold)),\n",
    "                 row=2, col=1)\n",
    "\n",
    "fig.update_layout({\n",
    "  \"annotations\": [\n",
    "    {\n",
    "      \"x\": 0.5, \n",
    "      \"y\": 1.0, \n",
    "      \"font\": {\"size\": 14}, \n",
    "      \"text\": \"Train / valid losses\", \n",
    "      \"xref\": \"paper\", \n",
    "      \"yref\": \"paper\", \n",
    "      \"xanchor\": \"center\", \n",
    "      \"yanchor\": \"bottom\", \n",
    "      \"showarrow\": False\n",
    "    }, \n",
    "    {\n",
    "      \"x\": 0.5, \n",
    "      \"y\": 0.5, \n",
    "      \"font\": {\"size\": 14}, \n",
    "      \"text\": \"Train / valid AUC\", \n",
    "      \"xref\": \"paper\", \n",
    "      \"yref\": \"paper\", \n",
    "      \"xanchor\": \"center\", \n",
    "      \"yanchor\": \"bottom\", \n",
    "      \"showarrow\": False\n",
    "    }, \n",
    "  ]\n",
    "})\n",
    "\n",
    "fig.show()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}