entity-tagging/4.concat-bahdanau-char-ngrams.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from bs4 import BeautifulSoup\n",
    "import re\n",
    "import numpy as np\n",
    "import tensorflow as tf\n",
    "from tqdm import tqdm\n",
    "import itertools"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "def process_string(string):\n",
    "    string = re.sub('[^A-Za-z0-9\\-\\/ ]+', ' ', string).split()\n",
    "    return [y.strip() for y in string]\n",
    "\n",
    "def to_title(string):\n",
    "    if string.isupper():\n",
    "        string = string.title()\n",
    "    return string"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "def parse_raw(filename):\n",
    "    with open(filename, 'r') as fopen:\n",
    "        entities = fopen.read()\n",
    "    soup = BeautifulSoup(entities, 'html.parser')\n",
    "    inside_tag = ''\n",
    "    texts, labels = [], []\n",
    "    for sentence in soup.prettify().split('\\n'):\n",
    "        if len(inside_tag):\n",
    "            splitted = process_string(sentence)\n",
    "            texts += splitted\n",
    "            labels += [inside_tag] * len(splitted)\n",
    "            inside_tag = ''\n",
    "        else:\n",
    "            if not sentence.find('</'):\n",
    "                pass\n",
    "            elif not sentence.find('<'):\n",
    "                inside_tag = sentence.split('>')[0][1:]\n",
    "            else:\n",
    "                splitted = process_string(sentence)\n",
    "                texts += splitted\n",
    "                labels += ['OTHER'] * len(splitted)\n",
    "    assert (len(texts)==len(labels)), \"length texts and labels are not same\"\n",
    "    print('len texts and labels: ', len(texts))\n",
    "    return texts,labels"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "len texts and labels:  34012\n",
      "len texts and labels:  9249\n"
     ]
    }
   ],
   "source": [
    "train_texts, train_labels = parse_raw('data_train.txt')\n",
    "test_texts, test_labels = parse_raw('data_test.txt')\n",
    "train_texts += test_texts\n",
    "train_labels += test_labels"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('entities-bm-normalize-v3.txt','r') as fopen:\n",
    "    entities_bm = fopen.read().split('\\n')[:-1]\n",
    "entities_bm = [i.split() for i in entities_bm]\n",
    "entities_bm = [[i[0],'TIME' if i[0] in 'jam' else i[1]] for i in entities_bm]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "replace_by = {'organizaiton':'organization','orgnization':'organization',\n",
    "             'othoer': 'OTHER'}\n",
    "\n",
    "with open('NER-part1.txt','r') as fopen:\n",
    "    nexts = fopen.read().split('\\n')[:-1]\n",
    "nexts = [i.split() for i in nexts]\n",
    "for i in nexts:\n",
    "    if len(i) == 2:\n",
    "        label = i[1].lower()\n",
    "        if 'other' in label:\n",
    "            label = label.upper()\n",
    "        if label in replace_by:\n",
    "            label = replace_by[label]\n",
    "        train_labels.append(label)\n",
    "        train_texts.append(i[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "'KN'\n",
      "'KA'\n"
     ]
    }
   ],
   "source": [
    "replace_by = {'LOC':'location','PRN':'person','NORP':'organization','ORG':'organization','LAW':'law',\n",
    "             'EVENT':'event','FAC':'organization','TIME':'time','O':'OTHER','ART':'person','DOC':'law'}\n",
    "for i in entities_bm:\n",
    "    try:\n",
    "        string = process_string(i[0])\n",
    "        if len(string):\n",
    "            train_labels.append(replace_by[i[1]])\n",
    "            train_texts.append(process_string(i[0])[0])  \n",
    "    except Exception as e:\n",
    "        print(e)\n",
    "        \n",
    "assert (len(train_texts)==len(train_labels)), \"length texts and labels are not same\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(array(['OTHER', 'event', 'law', 'location', 'organization', 'person',\n",
       "        'quantity', 'time'], dtype='<U12'),\n",
       " array([49712,   234,   185,  2056,  2596,  4397,  1341,  1296]))"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.unique(train_labels,return_counts=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "def _pad_sequence(\n",
    "    sequence,\n",
    "    n,\n",
    "    pad_left = False,\n",
    "    pad_right = False,\n",
    "    left_pad_symbol = None,\n",
    "    right_pad_symbol = None,\n",
    "):\n",
    "    sequence = iter(sequence)\n",
    "    if pad_left:\n",
    "        sequence = itertools.chain((left_pad_symbol,) * (n - 1), sequence)\n",
    "    if pad_right:\n",
    "        sequence = itertools.chain(sequence, (right_pad_symbol,) * (n - 1))\n",
    "    return sequence\n",
    "\n",
    "\n",
    "def ngrams(\n",
    "    sequence,\n",
    "    n,\n",
    "    pad_left = False,\n",
    "    pad_right = False,\n",
    "    left_pad_symbol = None,\n",
    "    right_pad_symbol = None,\n",
    "):\n",
    "    \"\"\"\n",
    "    generate ngrams\n",
    "\n",
    "    Parameters\n",
    "    ----------\n",
    "    sequence : list of str\n",
    "        list of tokenize words\n",
    "    n : int\n",
    "        ngram size\n",
    "\n",
    "    Returns\n",
    "    -------\n",
    "    ngram: list\n",
    "    \"\"\"\n",
    "    sequence = _pad_sequence(\n",
    "        sequence, n, pad_left, pad_right, left_pad_symbol, right_pad_symbol\n",
    "    )\n",
    "\n",
    "    history = []\n",
    "    while n > 1:\n",
    "        try:\n",
    "            next_item = next(sequence)\n",
    "        except StopIteration:\n",
    "            return\n",
    "        history.append(next_item)\n",
    "        n -= 1\n",
    "    for item in sequence:\n",
    "        history.append(item)\n",
    "        yield tuple(history)\n",
    "        del history[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_ngrams(s, grams=(2,3,4)):\n",
    "    return [''.join(i) for k in grams for i in list(ngrams(s,k))]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "word2idx = {'PAD': 0,'NUM':1,'UNK':2}\n",
    "tag2idx = {'PAD': 0}\n",
    "char2idx = {'PAD': 0,'NUM':1,'UNK':2}\n",
    "word_idx = 3\n",
    "tag_idx = 1\n",
    "char_idx = 3\n",
    "\n",
    "def parse_XY(texts, labels):\n",
    "    global word2idx, tag2idx, char2idx, word_idx, tag_idx, char_idx\n",
    "    X, Y = [], []\n",
    "    for no, text in enumerate(texts):\n",
    "        text = text.lower()\n",
    "        if len(text) < 2:\n",
    "            continue\n",
    "        tag = labels[no]\n",
    "        for c in get_ngrams(text):\n",
    "            if c not in char2idx:\n",
    "                char2idx[c] = char_idx\n",
    "                char_idx += 1\n",
    "        if tag not in tag2idx:\n",
    "            tag2idx[tag] = tag_idx\n",
    "            tag_idx += 1\n",
    "        Y.append(tag2idx[tag])\n",
    "        if text not in word2idx:\n",
    "            word2idx[text] = word_idx\n",
    "            word_idx += 1\n",
    "        X.append(word2idx[text])\n",
    "    return X, np.array(Y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "X, Y = parse_XY(train_texts, train_labels)\n",
    "idx2word={idx: tag for tag, idx in word2idx.items()}\n",
    "idx2tag = {i: w for w, i in tag2idx.items()}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "seq_len = 50\n",
    "def iter_seq(x):\n",
    "    return np.array([x[i: i+seq_len] for i in range(0, len(x)-seq_len, 1)])\n",
    "\n",
    "def to_train_seq(*args):\n",
    "    return [iter_seq(x) for x in args]\n",
    "\n",
    "def generate_char_seq(batch):\n",
    "    x = [[len(get_ngrams(idx2word[i])) for i in k] for k in batch]\n",
    "    maxlen = max([j for i in x for j in i])\n",
    "    temp = np.zeros((batch.shape[0],batch.shape[1], maxlen),dtype=np.int32)\n",
    "    for i in range(batch.shape[0]):\n",
    "        for k in range(batch.shape[1]):\n",
    "            for no, c in enumerate(get_ngrams(idx2word[batch[i,k]])):\n",
    "                temp[i,k,no] = char2idx[c]\n",
    "    return temp"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(61353, 50)"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_seq, Y_seq = to_train_seq(X, Y)\n",
    "X_char_seq = generate_char_seq(X_seq)\n",
    "X_seq.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(61353, 50, 63)"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_char_seq.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.6/dist-packages/sklearn/cross_validation.py:41: DeprecationWarning: This module was deprecated in version 0.18 in favor of the model_selection module into which all the refactored classes and functions are moved. Also note that the interface of the new CV iterators are different from that of this module. This module will be removed in 0.20.\n",
      "  \"This module will be removed in 0.20.\", DeprecationWarning)\n"
     ]
    }
   ],
   "source": [
    "from sklearn.cross_validation import train_test_split\n",
    "train_Y, test_Y, train_X, test_X = train_test_split(Y_seq, X_char_seq,test_size=0.2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Model:\n",
    "    def __init__(\n",
    "        self,\n",
    "        dim_word,\n",
    "        dropout,\n",
    "        learning_rate,\n",
    "        hidden_size_word,\n",
    "        num_layers,\n",
    "    ):\n",
    "        def cells(size, reuse = False):\n",
    "            return tf.contrib.rnn.DropoutWrapper(\n",
    "                tf.nn.rnn_cell.LSTMCell(\n",
    "                    size,\n",
    "                    initializer = tf.orthogonal_initializer(),\n",
    "                    reuse = reuse,\n",
    "                ),\n",
    "                output_keep_prob = dropout,\n",
    "            )\n",
    "\n",
    "        def bahdanau(embedded, size):\n",
    "            attention_mechanism = tf.contrib.seq2seq.BahdanauAttention(\n",
    "                num_units = hidden_size_word, memory = embedded\n",
    "            )\n",
    "            return tf.contrib.seq2seq.AttentionWrapper(\n",
    "                cell = cells(hidden_size_word),\n",
    "                attention_mechanism = attention_mechanism,\n",
    "                attention_layer_size = hidden_size_word,\n",
    "            )\n",
    "\n",
    "        self.word_ids = tf.placeholder(tf.int32, shape = [None, None, None])\n",
    "        self.labels = tf.placeholder(tf.int32, shape = [None, None])\n",
    "        self.maxlen = tf.shape(self.word_ids)[1]\n",
    "        self.lengths = tf.count_nonzero(tf.reduce_sum(self.word_ids,axis=2), 1)\n",
    "\n",
    "        self.word_embeddings = tf.Variable(\n",
    "            tf.truncated_normal(\n",
    "                [len(char2idx), dim_word], stddev = 1.0 / np.sqrt(dim_word)\n",
    "            )\n",
    "        )\n",
    "        word_embedded = tf.nn.embedding_lookup(\n",
    "            self.word_embeddings, self.word_ids\n",
    "        )\n",
    "        word_embedded = tf.reduce_mean(word_embedded, axis = 2)\n",
    "\n",
    "        for n in range(num_layers):\n",
    "            (out_fw, out_bw), (\n",
    "                state_fw,\n",
    "                state_bw,\n",
    "            ) = tf.nn.bidirectional_dynamic_rnn(\n",
    "                cell_fw = bahdanau(word_embedded, hidden_size_word),\n",
    "                cell_bw = bahdanau(word_embedded, hidden_size_word),\n",
    "                inputs = word_embedded,\n",
    "                dtype = tf.float32,\n",
    "                scope = 'bidirectional_rnn_word_%d' % (n),\n",
    "            )\n",
    "            word_embedded = tf.concat((out_fw, out_bw), 2)\n",
    "\n",
    "        logits = tf.layers.dense(word_embedded, len(idx2tag))\n",
    "        log_likelihood, transition_params = tf.contrib.crf.crf_log_likelihood(\n",
    "            logits, self.labels, self.lengths\n",
    "        )\n",
    "        self.cost = tf.reduce_mean(-log_likelihood)\n",
    "        self.optimizer = tf.train.AdamOptimizer(\n",
    "            learning_rate = learning_rate\n",
    "        ).minimize(self.cost)\n",
    "        mask = tf.sequence_mask(self.lengths, maxlen = self.maxlen)\n",
    "        self.tags_seq, tags_score = tf.contrib.crf.crf_decode(\n",
    "            logits, transition_params, self.lengths\n",
    "        )\n",
    "        self.tags_seq = tf.identity(self.tags_seq, name = 'logits')\n",
    "\n",
    "        self.prediction = tf.boolean_mask(self.tags_seq, mask)\n",
    "        mask_label = tf.boolean_mask(self.labels, mask)\n",
    "        correct_pred = tf.equal(self.prediction, mask_label)\n",
    "        correct_index = tf.cast(correct_pred, tf.float32)\n",
    "        self.accuracy = tf.reduce_mean(tf.cast(correct_pred, tf.float32))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.6/dist-packages/tensorflow/python/ops/gradients_impl.py:100: UserWarning: Converting sparse IndexedSlices to a dense Tensor of unknown shape. This may consume a large amount of memory.\n",
      "  \"Converting sparse IndexedSlices to a dense Tensor of unknown shape. \"\n"
     ]
    }
   ],
   "source": [
    "tf.reset_default_graph()\n",
    "sess = tf.InteractiveSession()\n",
    "\n",
    "dim_word = 128\n",
    "dropout = 0.8\n",
    "learning_rate = 1e-3\n",
    "hidden_size_word = 64\n",
    "num_layers = 2\n",
    "batch_size = 32\n",
    "\n",
    "model = Model(dim_word,dropout,learning_rate,hidden_size_word,num_layers)\n",
    "sess.run(tf.global_variables_initializer())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train minibatch loop: 100%|██████████| 1534/1534 [11:33<00:00,  2.23it/s, accuracy=0.971, cost=4.19]\n",
      "test minibatch loop: 100%|██████████| 384/384 [01:23<00:00,  4.45it/s, accuracy=0.989, cost=1.7] \n",
      "train minibatch loop:   0%|          | 0/1534 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "time taken: 777.0203394889832\n",
      "epoch: 0, training loss: 16.934028, training acc: 0.895233, valid loss: 2.954569, valid acc: 0.980174\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train minibatch loop: 100%|██████████| 1534/1534 [10:52<00:00,  4.14it/s, accuracy=0.995, cost=0.626]\n",
      "test minibatch loop: 100%|██████████| 384/384 [00:34<00:00, 11.29it/s, accuracy=0.996, cost=0.89] \n",
      "train minibatch loop:   0%|          | 0/1534 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "time taken: 686.5733714103699\n",
      "epoch: 1, training loss: 1.556880, training acc: 0.989226, valid loss: 0.898326, valid acc: 0.995229\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train minibatch loop: 100%|██████████| 1534/1534 [06:12<00:00,  4.15it/s, accuracy=0.995, cost=0.727]\n",
      "test minibatch loop: 100%|██████████| 384/384 [00:33<00:00, 11.42it/s, accuracy=0.997, cost=0.366]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "time taken: 406.7033176422119\n",
      "epoch: 2, training loss: 0.673882, training acc: 0.995572, valid loss: 0.536794, valid acc: 0.997698\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "import time\n",
    "\n",
    "for e in range(3):\n",
    "    lasttime = time.time()\n",
    "    train_acc, train_loss, test_acc, test_loss = 0, 0, 0, 0\n",
    "    pbar = tqdm(\n",
    "        range(0, len(train_X), batch_size), desc = 'train minibatch loop'\n",
    "    )\n",
    "    for i in pbar:\n",
    "        batch_x = train_X[i : min(i + batch_size, train_X.shape[0])]\n",
    "        batch_y = train_Y[i : min(i + batch_size, train_X.shape[0])]\n",
    "        acc, cost, _ = sess.run(\n",
    "            [model.accuracy, model.cost, model.optimizer],\n",
    "            feed_dict = {\n",
    "                model.word_ids: batch_x,\n",
    "                model.labels: batch_y\n",
    "            },\n",
    "        )\n",
    "        assert not np.isnan(cost)\n",
    "        train_loss += cost\n",
    "        train_acc += acc\n",
    "        pbar.set_postfix(cost = cost, accuracy = acc)\n",
    "    \n",
    "    pbar = tqdm(\n",
    "        range(0, len(test_X), batch_size), desc = 'test minibatch loop'\n",
    "    )\n",
    "    for i in pbar:\n",
    "        batch_x = test_X[i : min(i + batch_size, test_X.shape[0])]\n",
    "        batch_y = test_Y[i : min(i + batch_size, test_X.shape[0])]\n",
    "        acc, cost = sess.run(\n",
    "            [model.accuracy, model.cost],\n",
    "            feed_dict = {\n",
    "                model.word_ids: batch_x,\n",
    "                model.labels: batch_y\n",
    "            },\n",
    "        )\n",
    "        assert not np.isnan(cost)\n",
    "        test_loss += cost\n",
    "        test_acc += acc\n",
    "        pbar.set_postfix(cost = cost, accuracy = acc)\n",
    "    \n",
    "    train_loss /= len(train_X) / batch_size\n",
    "    train_acc /= len(train_X) / batch_size\n",
    "    test_loss /= len(test_X) / batch_size\n",
    "    test_acc /= len(test_X) / batch_size\n",
    "\n",
    "    print('time taken:', time.time() - lasttime)\n",
    "    print(\n",
    "        'epoch: %d, training loss: %f, training acc: %f, valid loss: %f, valid acc: %f\\n'\n",
    "        % (e, train_loss, train_acc, test_loss, test_acc)\n",
    "    )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "string = 'KUALA LUMPUR: Sempena sambutan Aidilfitri minggu depan, Perdana Menteri Tun Dr Mahathir Mohamad dan Menteri Pengangkutan Anthony Loke Siew Fook menitipkan pesanan khas kepada orang ramai yang mahu pulang ke kampung halaman masing-masing. Dalam video pendek terbitan Jabatan Keselamatan Jalan Raya (JKJR) itu, Dr Mahathir menasihati mereka supaya berhenti berehat dan tidur sebentar  sekiranya mengantuk ketika memandu.'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "def generate_char_seq(batch, UNK = 2):\n",
    "    maxlen_c = max([len(k) for k in batch])\n",
    "    x = [[len(get_ngrams(i)) for i in k] for k in batch]\n",
    "    maxlen = max([j for i in x for j in i])\n",
    "    temp = np.zeros((len(batch),maxlen_c,maxlen),dtype=np.int32)\n",
    "    for i in range(len(batch)):\n",
    "        for k in range(len(batch[i])):\n",
    "            for no, c in enumerate(get_ngrams(batch[i][k])):\n",
    "                temp[i,k,no] = char2idx.get(c, UNK)\n",
    "    return temp\n",
    "\n",
    "sequence = process_string(string.lower())\n",
    "X_char_seq = generate_char_seq([sequence])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(1, 57, 33)"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_char_seq.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "kuala location\n",
      "lumpur law\n",
      "sempena law\n",
      "sambutan law\n",
      "aidilfitri law\n",
      "minggu time\n",
      "depan time\n",
      "perdana person\n",
      "menteri person\n",
      "tun person\n",
      "dr person\n",
      "mahathir person\n",
      "mohamad person\n",
      "dan OTHER\n",
      "menteri organization\n",
      "pengangkutan organization\n",
      "anthony person\n",
      "loke person\n",
      "siew person\n",
      "fook person\n",
      "menitipkan OTHER\n",
      "pesanan OTHER\n",
      "khas OTHER\n",
      "kepada OTHER\n",
      "orang OTHER\n",
      "ramai OTHER\n",
      "yang OTHER\n",
      "mahu OTHER\n",
      "pulang OTHER\n",
      "ke OTHER\n",
      "kampung time\n",
      "halaman time\n",
      "masing-masing OTHER\n",
      "dalam OTHER\n",
      "video OTHER\n",
      "pendek OTHER\n",
      "terbitan OTHER\n",
      "jabatan organization\n",
      "keselamatan organization\n",
      "jalan organization\n",
      "raya organization\n",
      "jkjr person\n",
      "itu OTHER\n",
      "dr person\n",
      "mahathir person\n",
      "menasihati OTHER\n",
      "mereka OTHER\n",
      "supaya OTHER\n",
      "berhenti OTHER\n",
      "berehat location\n",
      "dan OTHER\n",
      "tidur OTHER\n",
      "sebentar OTHER\n",
      "sekiranya OTHER\n",
      "mengantuk OTHER\n",
      "ketika OTHER\n",
      "memandu OTHER\n"
     ]
    }
   ],
   "source": [
    "predicted = sess.run(model.tags_seq,\n",
    "            feed_dict = {\n",
    "                model.word_ids: X_char_seq,\n",
    "            })[0]\n",
    "for i in range(len(predicted)):\n",
    "    print(sequence[i],idx2tag[predicted[i]])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}