inference.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "1516987e",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "c:\\Users\\Safwan\\AppData\\Local\\Programs\\Python\\Python37\\lib\\site-packages\\requests\\__init__.py:80: RequestsDependencyWarning: urllib3 (1.26.5) or chardet (3.0.4) doesn't match a supported version!\n",
      "  RequestsDependencyWarning)\n"
     ]
    }
   ],
   "source": [
    "from torch.utils.data import Dataset, DataLoader\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.nn.functional as F\n",
    "import torch.optim as optim\n",
    "import jamotools\n",
    "import re\n",
    "import gluonnlp as nlp\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "from scipy import stats\n",
    "from sklearn.metrics import precision_score\n",
    "from sklearn.metrics import recall_score\n",
    "from sklearn.metrics import f1_score\n",
    "from gluonnlp.data import SentencepieceTokenizer\n",
    "from kobert.utils import get_tokenizer\n",
    "from kobert.utils import get_tokenizer\n",
    "from kobert.pytorch_kobert import get_pytorch_kobert_model\n",
    "from transformers import AdamW\n",
    "import gluonnlp as nlp\n",
    "import numpy as np\n",
    "import random"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "f5443dca",
   "metadata": {},
   "outputs": [],
   "source": [
    "class CharDataset(Dataset):\n",
    "    def __init__(self, dataset, sent_idx, label_idx, mode):\n",
    "        self.sentences = [line[sent_idx] for line in dataset]\n",
    "        self.labels = [int(line[label_idx]) for line in dataset]\n",
    "        self.korean = re.compile('[^1!ㄱ-ㅣ가-힣]+')\n",
    "        self.mode = mode\n",
    "        tok_path = '.cache/kobert_news_wiki_ko_cased-1087f8699e.spiece'\n",
    "        self.sp = SentencepieceTokenizer(tok_path)\n",
    "        self.vocab = self.make_vocab()\n",
    "        self.vocab_size = len(self.vocab)\n",
    "        self.q3 = self.get_q3()\n",
    "        self.char2idx = {u:i for i, u in enumerate(self.vocab)}\n",
    "        self.idx2char = {i:u for i, u in enumerate(self.vocab)}\n",
    "        self.max_len = self.find_max_len()\n",
    "        \n",
    "    def __getitem__(self, i):\n",
    "        return (self.preprocess_sentence(self.sentences[i]), torch.tensor(self.labels[i]).to(torch.float32))\n",
    "    \n",
    "    def __len__(self):\n",
    "        return len(self.labels)\n",
    "    \n",
    "    def make_vocab(self):\n",
    "        vocab = ''\n",
    "        for sentence in self.sentences:\n",
    "            vocab+=' '+sentence\n",
    "        vocab = self.make_token(vocab)\n",
    "        vocab = set(vocab)\n",
    "        vocab = sorted(vocab)\n",
    "        vocab.append('<UNK>') #######\n",
    "        vocab.append('<PAD>')\n",
    "        return vocab\n",
    "    \n",
    "    def make_token(self, sentence):\n",
    "        if self.mode == 'jamo':\n",
    "            chars = self.korean.sub('', jamotools.split_syllables(sentence))\n",
    "            return list(chars)\n",
    "        elif self.mode == 'char':\n",
    "            chars = self.korean.sub('', sentence)\n",
    "            return list(chars)\n",
    "        elif self.mode == 'sentencepiece':\n",
    "            return self.sp(sentence)\n",
    "        \n",
    "    def preprocess_sentence(self, sentence):\n",
    "        chars = self.make_token(sentence)\n",
    "        if len(chars) < self.q3:\n",
    "            need_pad = self.q3 - len(chars)\n",
    "            chars.extend(['<PAD>']*need_pad)\n",
    "        else:\n",
    "            chars = chars[:self.q3]\n",
    "        chars = torch.tensor([self.char2idx[x] for x in chars]).to(torch.int64)\n",
    "        return chars\n",
    "    \n",
    "    def find_max_len(self):\n",
    "        return max(len(self.make_token(item)) for item in self.sentences)\n",
    "    \n",
    "    def find_max_idx(self):\n",
    "        return self.sentences[np.argmax([len(self.make_token(item)) for item in self.sentences])]\n",
    "\n",
    "    \n",
    "    def get_q3(self):\n",
    "        values = np.array([len(self.make_token(x)) for x in self.sentences])\n",
    "        return int(np.quantile(values, 0.75))\n",
    "    \n",
    "    \n",
    "    def plot_len(self):\n",
    "        values = np.array([len(self.make_token(x)) for x in self.sentences])\n",
    "        plt.hist(values, density=True, bins=80)\n",
    "        plt.ylabel('count')\n",
    "        plt.xlabel('length of sequence')\n",
    "        plt.show()\n",
    "        print('문장 최대 길이 :',self.max_len)\n",
    "        results = stats.describe(values)\n",
    "        print('min={}, max={}, mean={}, Q2={} Q3={}'.format(results[1][0], results[1][1], results[2],\n",
    "                                                          np.median(values), np.quantile(values, 0.75)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "6c1d484a",
   "metadata": {},
   "outputs": [],
   "source": [
    "class Conv1d(nn.Module):\n",
    "    def __init__(self, vocab_size, seq_len, filters, embedding_dim, num_of_kernel):\n",
    "        super().__init__()\n",
    "        self.filters=filters\n",
    "        self.dropout_prob = 0.5\n",
    "        self.embedding_dim = embedding_dim\n",
    "        self.num_of_kernel = num_of_kernel\n",
    "        self.embedding = nn.Embedding(vocab_size, self.embedding_dim)\n",
    "        self.bn1 = nn.BatchNorm1d(self.num_of_kernel)\n",
    "        for i in range(len(self.filters)):\n",
    "            conv = nn.Conv1d(in_channels=self.embedding_dim, out_channels=self.num_of_kernel, kernel_size=self.filters[i])\n",
    "            setattr(self, f'conv_{i}', conv)\n",
    "        \n",
    "        self.dropout = nn.Dropout(p=self.dropout_prob)\n",
    "        self.bn2 = nn.BatchNorm1d(self.num_of_kernel*len(self.filters))\n",
    "        self.classifier = nn.Sequential(\n",
    "            nn.Linear(self.num_of_kernel*len(self.filters), 1),\n",
    "            nn.Sigmoid()\n",
    "        )\n",
    "        \n",
    "    def get_conv(self, i):\n",
    "        return getattr(self, f'conv_{i}')\n",
    "    \n",
    "    def forward(self, inp):\n",
    "        x = self.embedding(inp)\n",
    "        x = x.permute(0, 2, 1) ### embedding 을 transpose해줘야함.안하면 1d-conv이 seq 방향이 아닌, 임베딩 방향으로 진행됨.\n",
    "        conv_results = [\n",
    "            F.relu(self.bn1(self.get_conv(i)(x))).permute(0,2,1).max(1)[0]\n",
    "        for i in range(len(self.filters))]\n",
    "        x = torch.cat(conv_results, 1)\n",
    "        x = self.classifier(x)\n",
    "        x = x.squeeze()\n",
    "        \n",
    "        return x"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "ebd43fc5",
   "metadata": {},
   "outputs": [],
   "source": [
    "class GRU(nn.Module):\n",
    "    def __init__(self, vocab_size, seq_len, embedding_dim, hidden_size, n_layers):\n",
    "        super().__init__()\n",
    "        self.dropout_prob = 0.5\n",
    "        self.embedding_dim = embedding_dim\n",
    "        self.bidirectional = 0\n",
    "        self.n_layers = n_layers\n",
    "        self.hidden_size = hidden_size\n",
    "        self.embedding = nn.Embedding(vocab_size, self.embedding_dim)\n",
    "        self.rnn = nn.GRU(self.embedding_dim, self.hidden_size, num_layers=self.n_layers, dropout=self.dropout_prob, batch_first=True)\n",
    "        self.classifier = nn.Sequential(\n",
    "            nn.Linear(self.hidden_size, 1),\n",
    "            #nn.Linear(hidden_size*(self.bidirectional+1)*self.n_layers, 1),\n",
    "            nn.Sigmoid()\n",
    "        )\n",
    "        \n",
    "    \n",
    "    def forward(self, inp):\n",
    "        x = self.embedding(inp)\n",
    "        h_0 = self._init_state(batch_size=x.size(0))\n",
    "        x, _ = self.rnn(x, h_0)\n",
    "        h_t = x[:,-1,:]\n",
    "        output = self.classifier(h_t)\n",
    "        return output\n",
    "    \n",
    "    def _init_state(self, batch_size=1):\n",
    "        weight = next(self.parameters()).data\n",
    "        return weight.new(self.n_layers, batch_size, self.hidden_size).zero_()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "367987fb",
   "metadata": {},
   "outputs": [],
   "source": [
    "class BERTDataset(Dataset):\n",
    "    def __init__(self, dataset, sent_idx, label_idx, bert_tokenizer, max_len,\n",
    "                 pad, pair):\n",
    "        transform = nlp.data.BERTSentenceTransform(\n",
    "            bert_tokenizer, max_seq_length=max_len, pad=pad, pair=pair)\n",
    "\n",
    "        self.sentences = [transform([i[sent_idx]]) for i in dataset]\n",
    "        self.labels = [np.int32(i[label_idx]) for i in dataset]\n",
    "\n",
    "    def __getitem__(self, i):\n",
    "        return (self.sentences[i] + (self.labels[i], ))\n",
    "\n",
    "    def __len__(self):\n",
    "        return (len(self.labels))\n",
    "    \n",
    "class BERTClassifier(nn.Module):\n",
    "    def __init__(self,\n",
    "                 bert,\n",
    "                 hidden_size = 768,\n",
    "                 num_classes=1,\n",
    "                 dr_rate=None,\n",
    "                 params=None):\n",
    "        super(BERTClassifier, self).__init__()\n",
    "        self.bert = bert\n",
    "        self.dr_rate = dr_rate\n",
    "                 \n",
    "        self.classifier = nn.Sequential(\n",
    "          nn.Linear(hidden_size , num_classes),\n",
    "          nn.Sigmoid()\n",
    "        )\n",
    "        if dr_rate:\n",
    "            self.dropout = nn.Dropout(p=dr_rate)\n",
    "    \n",
    "    def gen_attention_mask(self, token_ids, valid_length):\n",
    "        attention_mask = torch.zeros_like(token_ids)\n",
    "        for i, v in enumerate(valid_length):\n",
    "            attention_mask[i][:v] = 1\n",
    "        return attention_mask.float()\n",
    "\n",
    "    def forward(self, token_ids, valid_length, segment_ids):\n",
    "        attention_mask = self.gen_attention_mask(token_ids, valid_length)\n",
    "        \n",
    "        _, pooler = self.bert(input_ids = token_ids, token_type_ids = segment_ids.long(), attention_mask = attention_mask.float().to(token_ids.device), return_dict=False)\n",
    "        if self.dr_rate:\n",
    "            out = self.dropout(pooler)\n",
    "        return self.classifier(out)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "a8f97c3c",
   "metadata": {
    "collapsed": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "using cached model. c:\\Users\\Safwan\\Desktop\\Profanity\\nlp-filter-out\\.cache\\kobert_v1.zip\n",
      "using cached model. c:\\Users\\Safwan\\Desktop\\Profanity\\nlp-filter-out\\.cache\\kobert_news_wiki_ko_cased-1087f8699e.spiece\n",
      "using cached model. c:\\Users\\Safwan\\Desktop\\Profanity\\nlp-filter-out\\.cache\\kobert_news_wiki_ko_cased-1087f8699e.spiece\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "BERTClassifier(\n",
       "  (bert): BertModel(\n",
       "    (embeddings): BertEmbeddings(\n",
       "      (word_embeddings): Embedding(8002, 768, padding_idx=1)\n",
       "      (position_embeddings): Embedding(512, 768)\n",
       "      (token_type_embeddings): Embedding(2, 768)\n",
       "      (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "      (dropout): Dropout(p=0.1, inplace=False)\n",
       "    )\n",
       "    (encoder): BertEncoder(\n",
       "      (layer): ModuleList(\n",
       "        (0): BertLayer(\n",
       "          (attention): BertAttention(\n",
       "            (self): BertSelfAttention(\n",
       "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "            (output): BertSelfOutput(\n",
       "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "          )\n",
       "          (intermediate): BertIntermediate(\n",
       "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "            (intermediate_act_fn): GELUActivation()\n",
       "          )\n",
       "          (output): BertOutput(\n",
       "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "        )\n",
       "        (1): BertLayer(\n",
       "          (attention): BertAttention(\n",
       "            (self): BertSelfAttention(\n",
       "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "            (output): BertSelfOutput(\n",
       "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "          )\n",
       "          (intermediate): BertIntermediate(\n",
       "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "            (intermediate_act_fn): GELUActivation()\n",
       "          )\n",
       "          (output): BertOutput(\n",
       "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "        )\n",
       "        (2): BertLayer(\n",
       "          (attention): BertAttention(\n",
       "            (self): BertSelfAttention(\n",
       "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "            (output): BertSelfOutput(\n",
       "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "          )\n",
       "          (intermediate): BertIntermediate(\n",
       "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "            (intermediate_act_fn): GELUActivation()\n",
       "          )\n",
       "          (output): BertOutput(\n",
       "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "        )\n",
       "        (3): BertLayer(\n",
       "          (attention): BertAttention(\n",
       "            (self): BertSelfAttention(\n",
       "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "            (output): BertSelfOutput(\n",
       "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "          )\n",
       "          (intermediate): BertIntermediate(\n",
       "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "            (intermediate_act_fn): GELUActivation()\n",
       "          )\n",
       "          (output): BertOutput(\n",
       "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "        )\n",
       "        (4): BertLayer(\n",
       "          (attention): BertAttention(\n",
       "            (self): BertSelfAttention(\n",
       "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "            (output): BertSelfOutput(\n",
       "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "          )\n",
       "          (intermediate): BertIntermediate(\n",
       "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "            (intermediate_act_fn): GELUActivation()\n",
       "          )\n",
       "          (output): BertOutput(\n",
       "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "        )\n",
       "        (5): BertLayer(\n",
       "          (attention): BertAttention(\n",
       "            (self): BertSelfAttention(\n",
       "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "            (output): BertSelfOutput(\n",
       "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "          )\n",
       "          (intermediate): BertIntermediate(\n",
       "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "            (intermediate_act_fn): GELUActivation()\n",
       "          )\n",
       "          (output): BertOutput(\n",
       "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "        )\n",
       "        (6): BertLayer(\n",
       "          (attention): BertAttention(\n",
       "            (self): BertSelfAttention(\n",
       "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "            (output): BertSelfOutput(\n",
       "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "          )\n",
       "          (intermediate): BertIntermediate(\n",
       "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "            (intermediate_act_fn): GELUActivation()\n",
       "          )\n",
       "          (output): BertOutput(\n",
       "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "        )\n",
       "        (7): BertLayer(\n",
       "          (attention): BertAttention(\n",
       "            (self): BertSelfAttention(\n",
       "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "            (output): BertSelfOutput(\n",
       "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "          )\n",
       "          (intermediate): BertIntermediate(\n",
       "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "            (intermediate_act_fn): GELUActivation()\n",
       "          )\n",
       "          (output): BertOutput(\n",
       "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "        )\n",
       "        (8): BertLayer(\n",
       "          (attention): BertAttention(\n",
       "            (self): BertSelfAttention(\n",
       "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "            (output): BertSelfOutput(\n",
       "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "          )\n",
       "          (intermediate): BertIntermediate(\n",
       "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "            (intermediate_act_fn): GELUActivation()\n",
       "          )\n",
       "          (output): BertOutput(\n",
       "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "        )\n",
       "        (9): BertLayer(\n",
       "          (attention): BertAttention(\n",
       "            (self): BertSelfAttention(\n",
       "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "            (output): BertSelfOutput(\n",
       "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "          )\n",
       "          (intermediate): BertIntermediate(\n",
       "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "            (intermediate_act_fn): GELUActivation()\n",
       "          )\n",
       "          (output): BertOutput(\n",
       "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "        )\n",
       "        (10): BertLayer(\n",
       "          (attention): BertAttention(\n",
       "            (self): BertSelfAttention(\n",
       "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "            (output): BertSelfOutput(\n",
       "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "          )\n",
       "          (intermediate): BertIntermediate(\n",
       "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "            (intermediate_act_fn): GELUActivation()\n",
       "          )\n",
       "          (output): BertOutput(\n",
       "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "        )\n",
       "        (11): BertLayer(\n",
       "          (attention): BertAttention(\n",
       "            (self): BertSelfAttention(\n",
       "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "            (output): BertSelfOutput(\n",
       "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "          )\n",
       "          (intermediate): BertIntermediate(\n",
       "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "            (intermediate_act_fn): GELUActivation()\n",
       "          )\n",
       "          (output): BertOutput(\n",
       "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "        )\n",
       "      )\n",
       "    )\n",
       "    (pooler): BertPooler(\n",
       "      (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "      (activation): Tanh()\n",
       "    )\n",
       "  )\n",
       "  (classifier): Sequential(\n",
       "    (0): Linear(in_features=768, out_features=1, bias=True)\n",
       "    (1): Sigmoid()\n",
       "  )\n",
       "  (dropout): Dropout(p=0.5, inplace=False)\n",
       ")"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "device = torch.device('cuda')\n",
    "mode = ['char', 'jamo', 'sentencepiece']\n",
    "\n",
    "dataset_train = nlp.data.TSVDataset('./data/train_hate_dataset_v2.txt')\n",
    "data_train_char = CharDataset(dataset_train, 0, 1, mode=mode[0])\n",
    "data_train_jamo = CharDataset(dataset_train, 0, 1, mode=mode[1])\n",
    "data_train_sentencepiece = CharDataset(dataset_train, 0, 1, mode=mode[2])\n",
    "\n",
    "#Conv1d\n",
    "filters = [4,5,6]\n",
    "embedding_dim = 1000\n",
    "num_of_kernel = 100\n",
    "\n",
    "#GRU\n",
    "embedding_dim_GRU = 750\n",
    "hidden_size = 100\n",
    "n_layers = 2\n",
    "\n",
    "#Conv1d\n",
    "#conv_char = Conv1d(data_train_char.vocab_size, data_train_char.get_q3(), filters, embedding_dim, num_of_kernel).to(device)\n",
    "#conv_char.load_state_dict(torch.load('./check_point/1dcnn_char.pt'))\n",
    "#conv_char.eval()\n",
    "#conv_jamo = Conv1d(data_train_jamo.vocab_size, data_train_jamo.get_q3(), filters, embedding_dim, num_of_kernel).to(device)\n",
    "#conv_jamo.load_state_dict(torch.load('./check_point/1dcnn_jamo.pt'))\n",
    "#conv_jamo.eval()\n",
    "#conv_sentencepiece = Conv1d(data_train_sentencepiece.vocab_size, data_train_sentencepiece.get_q3(), filters, embedding_dim, num_of_kernel).to(device)\n",
    "#conv_sentencepiece.load_state_dict(torch.load('./check_point/1dcnn_sentencepiece.pt'))\n",
    "#conv_sentencepiece.eval()\n",
    "\n",
    "#GRU\n",
    "GRU_char = GRU(data_train_char.vocab_size, data_train_char.get_q3(), embedding_dim_GRU, hidden_size, n_layers).to(device)\n",
    "GRU_char.load_state_dict(torch.load('./check_point/gru_char.pt'))\n",
    "GRU_char.eval()\n",
    "GRU_jamo = GRU(data_train_jamo.vocab_size, data_train_jamo.get_q3(), embedding_dim_GRU, hidden_size, n_layers).to(device)\n",
    "GRU_jamo.load_state_dict(torch.load('./check_point/gru_jamo.pt'))\n",
    "GRU_jamo.eval()\n",
    "GRU_sentencepiece = GRU(data_train_sentencepiece.vocab_size, data_train_sentencepiece.get_q3(), embedding_dim_GRU, hidden_size, n_layers).to(device)\n",
    "GRU_sentencepiece.load_state_dict(torch.load('./check_point/gru_sentencepiece.pt'))\n",
    "GRU_sentencepiece.eval()\n",
    "\n",
    "#KoBERT\n",
    "max_len = 32\n",
    "bertmodel, vocab = get_pytorch_kobert_model()\n",
    "dataset_train = nlp.data.TSVDataset('./data/train_hate_dataset_v2.txt')\n",
    "tokenizer = get_tokenizer()\n",
    "tok = nlp.data.BERTSPTokenizer(tokenizer, vocab, lower=False)\n",
    "data_train = BERTDataset(dataset_train, 0, 1, tok, max_len, True, False)\n",
    "kobert_tokenizer = nlp.data.BERTSentenceTransform(\n",
    "            tok, max_seq_length=max_len, pad=True, pair=False)\n",
    "koBERT = BERTClassifier(bertmodel,  dr_rate=0.5)\n",
    "model_dict = koBERT.state_dict()\n",
    "checkpoint = torch.load(\"./check_point/koBERT_state.pt\")\n",
    "convert_keys = {}\n",
    "for k, v in checkpoint.items():\n",
    "    new_key_name = k.replace(\"module.\", '')\n",
    "    if new_key_name not in model_dict:\n",
    "        print(\"{} is not int model_dict\".format(new_key_name))\n",
    "        continue\n",
    "    convert_keys[new_key_name] = v\n",
    "koBERT.load_state_dict(convert_keys)\n",
    "koBERT.to(device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "d246c151",
   "metadata": {},
   "outputs": [],
   "source": [
    "models = {'GRU_jamo':GRU_jamo,\n",
    "          'GRU_char':GRU_char,\n",
    "          'GRU_sentencepiece':GRU_sentencepiece}\n",
    "          #'conv_jamo':conv_jamo,\n",
    "          #'conv_char':conv_char,\n",
    "          #'conv_sentencepiece':conv_sentencepiece}\n",
    "\n",
    "def inference_by_all_model(sentence, models):\n",
    "    for model_name, model in models.items():\n",
    "        mode = model_name.split('_')[-1]\n",
    "        inference(sentence, model_name, model, mode)\n",
    "\n",
    "def inference(sentence, model_name, model, mode):\n",
    "    with torch.no_grad():\n",
    "        korean = re.compile('[^1!ㄱ-ㅣ가-힣]+')\n",
    "        if mode == 'char':\n",
    "            chars = data_train_char.make_token(sentence)\n",
    "            q3 = data_train_char.get_q3()\n",
    "            char2idx = data_train_char.char2idx\n",
    "            idx2char = data_train_char.idx2char\n",
    "        elif mode == 'jamo':\n",
    "            chars = data_train_jamo.make_token(sentence)\n",
    "            q3 = data_train_jamo.get_q3()\n",
    "            char2idx = data_train_jamo.char2idx\n",
    "            idx2char = data_train_jamo.idx2char\n",
    "        elif mode == 'sentencepiece':\n",
    "            chars = data_train_sentencepiece.make_token(sentence)\n",
    "            q3 = data_train_sentencepiece.get_q3()\n",
    "            char2idx = data_train_sentencepiece.char2idx\n",
    "            idx2char = data_train_sentencepiece.idx2char\n",
    "        if len(chars) < q3:\n",
    "            need_pad = q3 - len(chars)\n",
    "            chars.extend(['<PAD>']*need_pad)\n",
    "        else:\n",
    "            chars = chars[:q3]\n",
    "        chars = torch.tensor([char2idx[x] for x in chars]).to(torch.int64)\n",
    "        #print(chars)\n",
    "        chars = chars.to(device)\n",
    "        chars = torch.unsqueeze(chars, 0)\n",
    "        prt = [idx2char[x.item()] for x in chars.squeeze()]\n",
    "        #print(prt)\n",
    "        #print(chars)\n",
    "        outputs = model(chars)\n",
    "        pred = int(outputs>0.5)\n",
    "        if pred:\n",
    "            pred = '욕설'\n",
    "        else:\n",
    "            pred = '정상'\n",
    "            \n",
    "\n",
    "        print('model : {} {} {:.3f}'.format(model_name, pred, outputs.item()))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "fd53186f",
   "metadata": {},
   "source": [
    "# Inference"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "id": "4d9f1dce",
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "sentence : ㄴ1ㅇHㅁ1\n",
      "model : KoBERT 정상 0.053\n",
      "model : GRU_jamo 정상 0.022\n",
      "model : GRU_char 정상 0.011\n",
      "model : GRU_sentencepiece 정상 0.010\n",
      "\n"
     ]
    }
   ],
   "source": [
    "import warnings\n",
    "warnings.filterwarnings(action='ignore')\n",
    "\n",
    "sentences = ['씨이이이팔']\n",
    "\n",
    "for sentence in sentences:\n",
    "    print('sentence :', sentence)\n",
    "    model_name = 'KoBERT'\n",
    "    tokenized = tok(sentence)\n",
    "    tokenized.insert(0, '[CLS]')\n",
    "    tokenized.append('[SEP]')\n",
    "    token_ids = [vocab.token_to_idx[x] for x in tokenized]\n",
    "    len_token_ids = len(token_ids)\n",
    "    if len_token_ids < max_len:\n",
    "        need_pad = max_len - len_token_ids\n",
    "        token_ids.extend([3]*need_pad)\n",
    "    else:\n",
    "        token_ids = token_ids[:max_len]\n",
    "        token_ids[-1] = 3\n",
    "    token_ids = torch.tensor(token_ids)\n",
    "\n",
    "    _, valid_length, segment_ids = kobert_tokenizer(sentence)\n",
    "    token_ids = torch.tensor(token_ids).long().unsqueeze(0).to(device)\n",
    "    segment_ids = torch.tensor(segment_ids).long().unsqueeze(0).to(device)\n",
    "    valid_length = torch.tensor(valid_length).unsqueeze(0).to(device)\n",
    "    koBERT.eval()\n",
    "    out = koBERT(token_ids, valid_length, segment_ids)\n",
    "    pred = int(out>0.5)\n",
    "    if pred:\n",
    "        pred = '욕설'\n",
    "    else:\n",
    "        pred = '정상'\n",
    "    print('model : {} {} {:.3f}'.format(model_name, pred, out.item()))\n",
    "\n",
    "    inference_by_all_model(sentence, models)\n",
    "    print()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "84581f6a",
   "metadata": {},
   "source": [
    "# 토크나이저 비교"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "169f2f2e",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['ㄱ', 'ㅂ', 'ㅜ', 'ㅌ', 'ㅓ', 'ㄴ', 'ㄲ', 'ㅏ', 'ㅈ', 'ㅣ']\n",
      "['ㄱ', '부', '터', 'ㄴ', '까', '지']\n",
      "['▁', 'ᄀ', '부터', '▁', 'ᄂ', '까지']\n"
     ]
    }
   ],
   "source": [
    "print(data_train_jamo.make_token('ㄱ부터 ㄴ까지'))\n",
    "print(data_train_char.make_token('ㄱ부터 ㄴ까지'))\n",
    "print(data_train_sentencepiece.make_token('ㄱ부터 ㄴ까지'))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3.7.0 64-bit",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.0"
  },
  "vscode": {
   "interpreter": {
    "hash": "8f2d011d9b420cfd2a4f7e08465ea0cee8fd2bc9898c96caed1ba79c043953c3"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}