pos-tagging/1.concat-bahdanau.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import json\n",
    "import re\n",
    "import numpy as np\n",
    "import tensorflow as tf\n",
    "from sklearn.model_selection import KFold\n",
    "from tqdm import tqdm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "def process_string(string):\n",
    "    string = re.sub('[^A-Za-z0-9\\-\\/ ]+', ' ', string).split()\n",
    "    return [to_title(y.strip()) for y in string]\n",
    "\n",
    "def to_title(string):\n",
    "    if string.isupper():\n",
    "        string = string.title()\n",
    "    return string"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('pos-data-v3.json','r') as fopen:\n",
    "    dataset = json.load(fopen)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "list index out of range ['%', '%', 'SYM']\n",
      "list index out of range ['%', '%', 'SYM']\n",
      "list index out of range ['*', '*', 'SYM']\n",
      "list index out of range ['뭘봐', '뭘봐', 'PROPN']\n",
      "list index out of range ['%', '%', 'SYM']\n",
      "list index out of range ['ひ', 'ひ', 'PROPN']\n",
      "list index out of range ['ヒ', 'ヒ', 'PROPN']\n",
      "list index out of range ['形聲', '形聲', 'NOUN']\n",
      "list index out of range ['°', '°', 'SYM']\n",
      "list index out of range ['汉', '汉', 'PROPN']\n",
      "list index out of range ['东', '东', 'PROPN']\n",
      "list index out of range ['王', '王', 'PROPN']\n",
      "list index out of range ['（', '（', 'PROPN']\n",
      "list index out of range ['伊', '伊', 'PROPN']\n",
      "list index out of range ['）', '）', 'PROPN']\n",
      "list index out of range ['ȝ', 'ȝ', 'PROPN']\n",
      "list index out of range ['%', '%', 'SYM']\n",
      "list index out of range ['°', '°', 'SYM']\n",
      "list index out of range ['%', '%', 'SYM']\n",
      "list index out of range [\"'\", '_', 'PROPN']\n",
      "list index out of range ['碁', '碁', 'NOUN']\n",
      "list index out of range ['囲碁', '囲碁', 'NOUN']\n",
      "list index out of range ['*', '*', 'SYM']\n",
      "list index out of range ['*', '*', 'SYM']\n",
      "list index out of range ['*', '*', 'SYM']\n",
      "list index out of range ['*', '*', 'SYM']\n",
      "list index out of range ['*', '*', 'SYM']\n",
      "list index out of range ['*', '*', 'SYM']\n",
      "list index out of range ['*', '*', 'SYM']\n",
      "list index out of range ['*', '*', 'SYM']\n",
      "list index out of range ['*', '*', 'SYM']\n",
      "list index out of range ['*', '*', 'SYM']\n",
      "list index out of range ['会', '会', 'PROPN']\n",
      "list index out of range ['蔡武侯', '蔡武侯', 'PROPN']\n",
      "list index out of range ['%', '%', 'SYM']\n",
      "list index out of range ['%', '%', 'SYM']\n",
      "list index out of range ['%', '%', 'SYM']\n",
      "list index out of range ['+', '+', 'SYM']\n",
      "list index out of range ['+', '+', 'SYM']\n",
      "list index out of range ['*', '*', 'SYM']\n",
      "list index out of range ['*', '*', 'SYM']\n",
      "list index out of range ['*', '*', 'SYM']\n",
      "list index out of range ['*', '*', 'SYM']\n",
      "list index out of range ['*', '*', 'SYM']\n",
      "list index out of range ['*', '*', 'SYM']\n",
      "list index out of range ['*', '*', 'SYM']\n",
      "list index out of range ['*', '*', 'SYM']\n",
      "list index out of range ['*', '*', 'SYM']\n",
      "list index out of range ['*', '*', 'SYM']\n",
      "list index out of range ['*', '*', 'SYM']\n",
      "list index out of range ['*', '*', 'SYM']\n",
      "list index out of range ['*', '*', 'SYM']\n",
      "list index out of range ['*', '*', 'SYM']\n",
      "list index out of range ['*', '*', 'SYM']\n",
      "list index out of range ['*', '*', 'SYM']\n",
      "list index out of range ['*', '*', 'SYM']\n",
      "list index out of range ['*', '*', 'SYM']\n",
      "list index out of range ['*', '*', 'SYM']\n",
      "list index out of range ['%', '%', 'SYM']\n",
      "list index out of range ['%', '%', 'SYM']\n",
      "list index out of range ['+', '+', 'SYM']\n",
      "list index out of range ['%', '%', 'SYM']\n",
      "list index out of range ['%', '%', 'SYM']\n",
      "list index out of range ['~', '~', 'ADJ']\n",
      "list index out of range ['%', '%', 'SYM']\n",
      "list index out of range ['ОАО', 'оао', 'PROPN']\n",
      "list index out of range ['Газпром', 'газпром', 'PROPN']\n",
      "list index out of range [\"''\", '_', 'PROPN']\n",
      "list index out of range ['°', '°', 'SYM']\n",
      "list index out of range ['°', '°', 'SYM']\n",
      "list index out of range ['佐々木', '佐々木', 'PROPN']\n",
      "list index out of range ['功', '功', 'PROPN']\n",
      "list index out of range ['ɳ', 'ɳ', 'PROPN']\n",
      "list index out of range ['ʂ', 'ʂ', 'PROPN']\n",
      "list index out of range ['ʈ', 'ʈ', 'PROPN']\n",
      "list index out of range ['ɖ', 'ɖ', 'PROPN']\n",
      "list index out of range ['=', '=', 'SYM']\n",
      "list index out of range ['=', '=', 'SYM']\n",
      "list index out of range [\"'\", '_', 'NOUN']\n",
      "list index out of range [\"'\", '_', 'PROPN']\n",
      "list index out of range ['=', '=', 'SYM']\n",
      "list index out of range ['=', '=', 'SYM']\n",
      "list index out of range ['鉄の森', '鉄の森', 'PROPN']\n",
      "list index out of range ['アイゼンヴァルト', 'アイゼンヴァルト', 'PROPN']\n",
      "list index out of range ['=', '=', 'SYM']\n",
      "list index out of range ['=', '=', 'SYM']\n",
      "list index out of range ['=', '=', 'SYM']\n",
      "list index out of range ['=', '=', 'SYM']\n",
      "list index out of range ['+', '+', 'SYM']\n",
      "list index out of range ['=', '=', 'SYM']\n",
      "list index out of range ['=', '=', 'SYM']\n",
      "list index out of range ['=', '=', 'SYM']\n",
      "list index out of range ['=', '=', 'SYM']\n",
      "list index out of range ['=', '=', 'SYM']\n",
      "list index out of range ['%', '%', 'SYM']\n",
      "list index out of range ['壽', '壽', 'PROPN']\n",
      "list index out of range ['陽', '陽', 'PROPN']\n",
      "list index out of range ['裴叔業', '裴叔業', 'PROPN']\n",
      "list index out of range ['%', '%', 'SYM']\n",
      "list index out of range ['#', '#', 'SYM']\n",
      "list index out of range ['+', '+', 'SYM']\n",
      "list index out of range ['فيروز', 'فيروز', 'PROPN']\n",
      "list index out of range ['الديلامي', 'الديلامي', 'PROPN']\n",
      "list index out of range ['נביאים', 'נביאים', 'NOUN']\n",
      "list index out of range ['ראשונים', 'ראשונים', 'NOUN']\n",
      "list index out of range ['נביאים', 'נביאים', 'NOUN']\n",
      "list index out of range ['جامع', 'جامع', 'PROPN']\n",
      "list index out of range ['الرئيس', 'الرئيس', 'PROPN']\n",
      "list index out of range ['الصالح', 'الصالح', 'PROPN']\n",
      "list index out of range ['—', '—', 'NOUN']\n",
      "list index out of range ['°', '°', 'SYM']\n",
      "list index out of range ['°', '°', 'SYM']\n",
      "list index out of range ['%', '%', 'SYM']\n",
      "list index out of range ['ä', 'ä', 'PROPN']\n",
      "list index out of range ['ʈ', 'ʈ', 'PROPN']\n",
      "list index out of range ['ʈʰ', 'ʈʰ', 'PROPN']\n",
      "list index out of range ['°', '°', 'SYM']\n",
      "list index out of range ['°', '°', 'SYM']\n",
      "list index out of range [\"''\", '_', 'PROPN']\n",
      "list index out of range ['€', '€', 'SYM']\n",
      "list index out of range ['\\u200e', '\\u200e', 'VERB']\n",
      "list index out of range ['\\u200e', '\\u200e', 'NOUN']\n",
      "list index out of range ['%', '%', 'SYM']\n",
      "list index out of range ['%', '%', 'SYM']\n",
      "list index out of range ['%', '%', 'SYM']\n",
      "list index out of range ['%', '%', 'SYM']\n",
      "list index out of range ['%', '%', 'SYM']\n",
      "list index out of range ['$', '$', 'SYM']\n",
      "list index out of range ['Α', 'α', 'PROPN']\n",
      "list index out of range ['А', 'а', 'PROPN']\n",
      "list index out of range ['%', '%', 'SYM']\n",
      "list index out of range ['%', '%', 'SYM']\n",
      "list index out of range ['%', '%', 'SYM']\n",
      "list index out of range ['%', '%', 'SYM']\n",
      "list index out of range ['%', '%', 'SYM']\n",
      "list index out of range ['$', '$', 'SYM']\n",
      "list index out of range ['%', '%', 'SYM']\n",
      "list index out of range ['%', '%', 'SYM']\n",
      "list index out of range ['%', '%', 'SYM']\n",
      "list index out of range ['%', '%', 'SYM']\n",
      "list index out of range ['%', '%', 'SYM']\n",
      "list index out of range ['%', '%', 'SYM']\n",
      "list index out of range ['%', '%', 'SYM']\n",
      "list index out of range ['%', '%', 'SYM']\n",
      "list index out of range ['%', '%', 'SYM']\n",
      "list index out of range ['%', '%', 'SYM']\n",
      "list index out of range ['%', '%', 'SYM']\n",
      "list index out of range ['おる', 'おる', 'PROPN']\n",
      "list index out of range ['いる', 'いる', 'PROPN']\n",
      "list index out of range ['じゃ', 'じゃ', 'PROPN']\n",
      "list index out of range ['や', 'や', 'PROPN']\n",
      "list index out of range ['だ', 'だ', 'PROPN']\n",
      "list index out of range ['～', '～', 'PROPN']\n",
      "list index out of range ['ん', 'ん', 'PROPN']\n",
      "list index out of range ['わから', 'わから', 'PROPN']\n",
      "list index out of range ['ん', 'ん', 'PROPN']\n",
      "list index out of range ['～', '～', 'PROPN']\n",
      "list index out of range ['ない', 'ない', 'PROPN']\n",
      "list index out of range ['わからない', 'わからない', 'PROPN']\n",
      "list index out of range ['=', '=', 'SYM']\n",
      "list index out of range ['#', '#', 'SYM']\n",
      "list index out of range ['=', '=', 'SYM']\n",
      "list index out of range ['%', '%', 'SYM']\n",
      "list index out of range ['%', '%', 'SYM']\n",
      "list index out of range ['#', '#', 'SYM']\n",
      "list index out of range ['#', '#', 'SYM']\n",
      "list index out of range ['%', '%', 'SYM']\n",
      "list index out of range ['=', '=', 'SYM']\n",
      "list index out of range ['%', '%', 'SYM']\n",
      "list index out of range ['×', '×', 'NOUN']\n",
      "list index out of range ['°', '°', 'SYM']\n",
      "list index out of range ['–', '–', 'NUM']\n",
      "list index out of range ['°', '°', 'SYM']\n",
      "list index out of range ['続', '続', 'PROPN']\n",
      "list index out of range ['・', '・', 'PROPN']\n",
      "list index out of range ['三', '三', 'PROPN']\n",
      "list index out of range ['丁目', '丁目', 'PROPN']\n",
      "list index out of range ['の', 'の', 'PROPN']\n",
      "list index out of range ['夕日', '夕日', 'PROPN']\n",
      "list index out of range ['%', '%', 'SYM']\n",
      "list index out of range ['=', '=', 'SYM']\n",
      "list index out of range ['=', '=', 'SYM']\n",
      "list index out of range ['=', '=', 'SYM']\n",
      "list index out of range ['=', '=', 'SYM']\n",
      "list index out of range ['=', '=', 'SYM']\n",
      "list index out of range ['=', '=', 'SYM']\n",
      "list index out of range ['Ἑλλάδος', 'ἑλλάδος', 'PROPN']\n",
      "list index out of range ['παίδευσις', 'παίδευσις', 'PROPN']\n",
      "list index out of range ['€', '€', 'SYM']\n",
      "list index out of range ['ê', 'ê', 'PROPN']\n",
      "list index out of range ['é', 'é', 'PROPN']\n",
      "list index out of range ['è', 'è', 'PROPN']\n",
      "list index out of range ['ë', 'ë', 'PROPN']\n",
      "list index out of range ['ē', 'ē', 'PROPN']\n",
      "list index out of range ['ĕ', 'ĕ', 'PROPN']\n",
      "list index out of range ['ě', 'ě', 'PROPN']\n",
      "list index out of range ['ẽ', 'ẽ', 'PROPN']\n",
      "list index out of range ['ė', 'ė', 'PROPN']\n",
      "list index out of range ['ę', 'ę', 'PROPN']\n",
      "list index out of range ['ẻ', 'ẻ', 'PROPN']\n",
      "list index out of range ['ö', 'ö', 'PROPN']\n",
      "list index out of range ['°', '°', 'SYM']\n",
      "list index out of range ['°', '°', 'SYM']\n",
      "list index out of range ['°', '°', 'SYM']\n",
      "list index out of range ['°', '°', 'SYM']\n",
      "list index out of range ['.', '.', 'PROPN']\n",
      "list index out of range ['=', '=', 'SYM']\n",
      "list index out of range ['%', '%', 'SYM']\n",
      "list index out of range ['بندر', 'بندر', 'PROPN']\n",
      "list index out of range ['عباس', 'عباس', 'PROPN']\n",
      "list index out of range ['%', '%', 'SYM']\n",
      "list index out of range ['%', '%', 'SYM']\n",
      "list index out of range ['+', '+', 'SYM']\n",
      "list index out of range ['%', '%', 'SYM']\n",
      "list index out of range ['$', '$', 'SYM']\n",
      "list index out of range ['%', '%', 'SYM']\n",
      "list index out of range ['%', '%', 'SYM']\n",
      "list index out of range ['%', '%', 'SYM']\n",
      "list index out of range ['%', '%', 'SYM']\n",
      "list index out of range ['珠海', '珠海', 'PROPN']\n",
      "list index out of range [\"''\", '_', 'NOUN']\n",
      "list index out of range ['%', '%', 'SYM']\n",
      "list index out of range ['%', '%', 'SYM']\n",
      "list index out of range ['%', '%', 'SYM']\n",
      "list index out of range ['=', '=', 'SYM']\n",
      "list index out of range ['서종', '서종', 'PROPN']\n",
      "list index out of range ['제', '제', 'PROPN']\n",
      "list index out of range ['±', '±', 'SYM']\n",
      "list index out of range ['=', '=', 'SYM']\n",
      "list index out of range ['仙', '仙', 'PROPN']\n",
      "list index out of range ['仚', '仚', 'PROPN']\n",
      "list index out of range ['僊', '僊', 'PROPN']\n",
      "list index out of range ['=', '=', 'SYM']\n",
      "list index out of range ['=', '=', 'SYM']\n",
      "list index out of range ['腹切り', '腹切り', 'PROPN']\n",
      "list index out of range ['§', '§', 'PROPN']\n",
      "list index out of range ['%', '%', 'SYM']\n",
      "list index out of range ['%', '%', 'SYM']\n",
      "list index out of range ['%', '%', 'SYM']\n",
      "list index out of range ['%', '%', 'SYM']\n",
      "list index out of range ['%', '%', 'SYM']\n",
      "list index out of range ['%', '%', 'SYM']\n",
      "list index out of range ['%', '%', 'SYM']\n",
      "list index out of range ['%', '%', 'SYM']\n",
      "list index out of range ['=', '=', 'SYM']\n",
      "list index out of range ['육식', '육식', 'PROPN']\n",
      "list index out of range ['동물', '동물', 'PROPN']\n",
      "list index out of range ['도망자', '도망자', 'PROPN']\n",
      "list index out of range ['%', '%', 'SYM']\n",
      "list index out of range ['=', '=', 'SYM']\n",
      "list index out of range ['=', '=', 'SYM']\n",
      "list index out of range ['الغرب', 'الغرب', 'PROPN']\n",
      "list index out of range ['%', '%', 'SYM']\n",
      "list index out of range ['%', '%', 'SYM']\n",
      "list index out of range ['%', '%', 'SYM']\n",
      "list index out of range ['%', '%', 'SYM']\n",
      "list index out of range ['%', '%', 'SYM']\n",
      "list index out of range ['%', '%', 'SYM']\n",
      "list index out of range ['=', '=', 'SYM']\n",
      "list index out of range ['سدوم', 'سدوم', 'PROPN']\n",
      "list index out of range ['عمورة', 'عمورة', 'PROPN']\n",
      "list index out of range ['高雄', '高雄', 'PROPN']\n",
      "list index out of range ['大眾', '大眾', 'PROPN']\n",
      "list index out of range ['捷', '捷', 'PROPN']\n",
      "list index out of range ['運', '運', 'PROPN']\n",
      "list index out of range ['系統', '系統', 'PROPN']\n",
      "list index out of range ['°', '°', 'SYM']\n",
      "list index out of range ['°', '°', 'SYM']\n",
      "list index out of range ['=', '=', 'SYM']\n",
      "list index out of range ['亳', '亳', 'PROPN']\n",
      "list index out of range ['°', '°', 'SYM']\n",
      "list index out of range ['المعقبات', 'المعقبات', 'PROPN']\n",
      "list index out of range ['%', '%', 'SYM']\n",
      "list index out of range ['#', '#', 'SYM']\n",
      "list index out of range ['#', '#', 'SYM']\n",
      "list index out of range ['#', '#', 'SYM']\n",
      "list index out of range ['#', '#', 'SYM']\n",
      "list index out of range ['%', '%', 'SYM']\n"
     ]
    }
   ],
   "source": [
    "texts, labels = [], []\n",
    "for i in dataset:\n",
    "    try:\n",
    "        texts.append(process_string(i[0])[0].lower())\n",
    "        labels.append(i[-1])\n",
    "    except Exception as e:\n",
    "        print(e, i)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "word2idx = {'PAD': 0,'NUM':1,'UNK':2}\n",
    "tag2idx = {'PAD': 0}\n",
    "char2idx = {'PAD': 0}\n",
    "word_idx = 3\n",
    "tag_idx = 1\n",
    "char_idx = 1\n",
    "\n",
    "def parse_XY(texts, labels):\n",
    "    global word2idx, tag2idx, char2idx, word_idx, tag_idx, char_idx\n",
    "    X, Y = [], []\n",
    "    for no, text in enumerate(texts):\n",
    "        text = to_title(text)\n",
    "        tag = labels[no]\n",
    "        for c in text:\n",
    "            if c not in char2idx:\n",
    "                char2idx[c] = char_idx\n",
    "                char_idx += 1\n",
    "        if tag not in tag2idx:\n",
    "            tag2idx[tag] = tag_idx\n",
    "            tag_idx += 1\n",
    "        Y.append(tag2idx[tag])\n",
    "        if text not in word2idx:\n",
    "            word2idx[text] = word_idx\n",
    "            word_idx += 1\n",
    "        X.append(word2idx[text])\n",
    "    return X, np.array(Y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "X, Y = parse_XY(texts, labels)\n",
    "idx2word={idx: tag for tag, idx in word2idx.items()}\n",
    "idx2tag = {i: w for w, i in tag2idx.items()}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "seq_len = 50\n",
    "def iter_seq(x):\n",
    "    return np.array([x[i: i+seq_len] for i in range(0, len(x)-seq_len, 1)])\n",
    "\n",
    "def to_train_seq(*args):\n",
    "    return [iter_seq(x) for x in args]\n",
    "\n",
    "def generate_char_seq(batch):\n",
    "    x = [[len(idx2word[i]) for i in k] for k in batch]\n",
    "    maxlen = max([j for i in x for j in i])\n",
    "    temp = np.zeros((batch.shape[0],batch.shape[1],maxlen),dtype=np.int32)\n",
    "    for i in range(batch.shape[0]):\n",
    "        for k in range(batch.shape[1]):\n",
    "            for no, c in enumerate(idx2word[batch[i,k]]):\n",
    "                temp[i,k,-1-no] = char2idx[c]\n",
    "    return temp"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(103367, 50)"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_seq, Y_seq = to_train_seq(X, Y)\n",
    "X_char_seq = generate_char_seq(X_seq)\n",
    "X_seq.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "import json\n",
    "with open('bahdanau-pos.json','w') as fopen:\n",
    "    fopen.write(json.dumps({'idx2tag':idx2tag,'idx2word':idx2word,\n",
    "           'word2idx':word2idx,'tag2idx':tag2idx,'char2idx':char2idx}))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "from keras.utils import to_categorical\n",
    "Y_seq_3d = [to_categorical(i, num_classes=len(tag2idx)) for i in Y_seq]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.5/dist-packages/sklearn/cross_validation.py:41: DeprecationWarning: This module was deprecated in version 0.18 in favor of the model_selection module into which all the refactored classes and functions are moved. Also note that the interface of the new CV iterators are different from that of this module. This module will be removed in 0.20.\n",
      "  \"This module will be removed in 0.20.\", DeprecationWarning)\n"
     ]
    }
   ],
   "source": [
    "from sklearn.cross_validation import train_test_split\n",
    "train_X, test_X, train_Y, test_Y, train_char, test_char = train_test_split(X_seq, Y_seq_3d, X_char_seq, \n",
    "                                                                           test_size=0.1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Model:\n",
    "    def __init__(\n",
    "        self,\n",
    "        dim_word,\n",
    "        dim_char,\n",
    "        dropout,\n",
    "        learning_rate,\n",
    "        hidden_size_char,\n",
    "        hidden_size_word,\n",
    "        num_layers,\n",
    "    ):\n",
    "        def cells(size, reuse = False):\n",
    "            return tf.contrib.rnn.DropoutWrapper(\n",
    "                tf.nn.rnn_cell.LSTMCell(\n",
    "                    size,\n",
    "                    initializer = tf.orthogonal_initializer(),\n",
    "                    reuse = reuse,\n",
    "                ),\n",
    "                state_keep_prob = dropout,\n",
    "                output_keep_prob = dropout,\n",
    "            )\n",
    "\n",
    "        def bahdanau(embedded, size):\n",
    "            attention_mechanism = tf.contrib.seq2seq.BahdanauAttention(\n",
    "                num_units = hidden_size_word, memory = embedded\n",
    "            )\n",
    "            return tf.contrib.seq2seq.AttentionWrapper(\n",
    "                cell = cells(hidden_size_word),\n",
    "                attention_mechanism = attention_mechanism,\n",
    "                attention_layer_size = hidden_size_word,\n",
    "            )\n",
    "\n",
    "        self.word_ids = tf.placeholder(tf.int32, shape = [None, None])\n",
    "        self.char_ids = tf.placeholder(tf.int32, shape = [None, None, None])\n",
    "        self.labels = tf.placeholder(tf.int32, shape = [None, None, None])\n",
    "        self.maxlen = tf.shape(self.word_ids)[1]\n",
    "        self.lengths = tf.count_nonzero(self.word_ids, 1)\n",
    "\n",
    "        self.word_embeddings = tf.Variable(\n",
    "            tf.truncated_normal(\n",
    "                [len(word2idx), dim_word], stddev = 1.0 / np.sqrt(dim_word)\n",
    "            )\n",
    "        )\n",
    "        self.char_embeddings = tf.Variable(\n",
    "            tf.truncated_normal(\n",
    "                [len(char2idx), dim_char], stddev = 1.0 / np.sqrt(dim_char)\n",
    "            )\n",
    "        )\n",
    "\n",
    "        word_embedded = tf.nn.embedding_lookup(\n",
    "            self.word_embeddings, self.word_ids\n",
    "        )\n",
    "        char_embedded = tf.nn.embedding_lookup(\n",
    "            self.char_embeddings, self.char_ids\n",
    "        )\n",
    "        s = tf.shape(char_embedded)\n",
    "        char_embedded = tf.reshape(\n",
    "            char_embedded, shape = [s[0] * s[1], s[-2], dim_char]\n",
    "        )\n",
    "\n",
    "        for n in range(num_layers):\n",
    "            (out_fw, out_bw), (\n",
    "                state_fw,\n",
    "                state_bw,\n",
    "            ) = tf.nn.bidirectional_dynamic_rnn(\n",
    "                cell_fw = cells(hidden_size_char),\n",
    "                cell_bw = cells(hidden_size_char),\n",
    "                inputs = char_embedded,\n",
    "                dtype = tf.float32,\n",
    "                scope = 'bidirectional_rnn_char_%d' % (n),\n",
    "            )\n",
    "            char_embedded = tf.concat((out_fw, out_bw), 2)\n",
    "        output = tf.reshape(\n",
    "            char_embedded[:, -1], shape = [s[0], s[1], 2 * hidden_size_char]\n",
    "        )\n",
    "        word_embedded = tf.concat([word_embedded, output], axis = -1)\n",
    "\n",
    "        for n in range(num_layers):\n",
    "            (out_fw, out_bw), (\n",
    "                state_fw,\n",
    "                state_bw,\n",
    "            ) = tf.nn.bidirectional_dynamic_rnn(\n",
    "                cell_fw = bahdanau(word_embedded,hidden_size_word),\n",
    "                cell_bw = bahdanau(word_embedded,hidden_size_word),\n",
    "                inputs = word_embedded,\n",
    "                dtype = tf.float32,\n",
    "                scope = 'bidirectional_rnn_word_%d' % (n),\n",
    "            )\n",
    "            word_embedded = tf.concat((out_fw, out_bw), 2)\n",
    "\n",
    "        logits = tf.layers.dense(word_embedded, len(idx2tag))\n",
    "        y_t = tf.argmax(self.labels, 2)\n",
    "        log_likelihood, transition_params = tf.contrib.crf.crf_log_likelihood(\n",
    "            logits, y_t, self.lengths\n",
    "        )\n",
    "        self.cost = tf.reduce_mean(-log_likelihood)\n",
    "        self.optimizer = tf.train.AdamOptimizer(\n",
    "            learning_rate = learning_rate\n",
    "        ).minimize(self.cost)\n",
    "        mask = tf.sequence_mask(self.lengths, maxlen = self.maxlen)\n",
    "        self.tags_seq, tags_score = tf.contrib.crf.crf_decode(\n",
    "            logits, transition_params, self.lengths\n",
    "        )\n",
    "        self.tags_seq = tf.identity(self.tags_seq, name = 'logits')\n",
    "\n",
    "        y_t = tf.cast(y_t, tf.int32)\n",
    "        self.prediction = tf.boolean_mask(self.tags_seq, mask)\n",
    "        mask_label = tf.boolean_mask(y_t, mask)\n",
    "        correct_pred = tf.equal(self.prediction, mask_label)\n",
    "        correct_index = tf.cast(correct_pred, tf.float32)\n",
    "        self.accuracy = tf.reduce_mean(tf.cast(correct_pred, tf.float32))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.5/dist-packages/tensorflow/python/ops/gradients_impl.py:112: UserWarning: Converting sparse IndexedSlices to a dense Tensor of unknown shape. This may consume a large amount of memory.\n",
      "  \"Converting sparse IndexedSlices to a dense Tensor of unknown shape. \"\n"
     ]
    }
   ],
   "source": [
    "tf.reset_default_graph()\n",
    "sess = tf.InteractiveSession()\n",
    "\n",
    "dim_word = 128\n",
    "dim_char = 256\n",
    "dropout = 0.8\n",
    "learning_rate = 1e-3\n",
    "hidden_size_char = 64\n",
    "hidden_size_word = 64\n",
    "num_layers = 2\n",
    "batch_size = 32\n",
    "\n",
    "model = Model(dim_word,dim_char,dropout,learning_rate,hidden_size_char,hidden_size_word,num_layers)\n",
    "sess.run(tf.global_variables_initializer())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train minibatch loop: 100%|██████████| 2908/2908 [49:45<00:00,  1.03it/s, accuracy=1, cost=0.3]      \n",
      "test minibatch loop: 100%|██████████| 324/324 [02:42<00:00,  2.12it/s, accuracy=1, cost=0.323]    \n",
      "train minibatch loop:   0%|          | 0/2908 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "time taken: 3148.280516386032\n",
      "epoch: 0, training loss: 8.941798, training acc: 0.943038, valid loss: 0.878317, valid acc: 0.997585\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train minibatch loop: 100%|██████████| 2908/2908 [50:24<00:00,  1.06it/s, accuracy=1, cost=0.0999]    \n",
      "test minibatch loop: 100%|██████████| 324/324 [02:47<00:00,  2.02it/s, accuracy=1, cost=0.316]    "
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "time taken: 3192.1942698955536\n",
      "epoch: 1, training loss: 0.560824, training acc: 0.996827, valid loss: 0.479058, valid acc: 1.000074\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "import time\n",
    "\n",
    "for e in range(2):\n",
    "    lasttime = time.time()\n",
    "    train_acc, train_loss, test_acc, test_loss = 0, 0, 0, 0\n",
    "    pbar = tqdm(\n",
    "        range(0, len(train_X), batch_size), desc = 'train minibatch loop'\n",
    "    )\n",
    "    for i in pbar:\n",
    "        batch_x = train_X[i : min(i + batch_size, train_X.shape[0])]\n",
    "        batch_char = train_char[i : min(i + batch_size, train_X.shape[0])]\n",
    "        batch_y = train_Y[i : min(i + batch_size, train_X.shape[0])]\n",
    "        acc, cost, _ = sess.run(\n",
    "            [model.accuracy, model.cost, model.optimizer],\n",
    "            feed_dict = {\n",
    "                model.word_ids: batch_x,\n",
    "                model.char_ids: batch_char,\n",
    "                model.labels: batch_y\n",
    "            },\n",
    "        )\n",
    "        assert not np.isnan(cost)\n",
    "        train_loss += cost\n",
    "        train_acc += acc\n",
    "        pbar.set_postfix(cost = cost, accuracy = acc)\n",
    "    \n",
    "    pbar = tqdm(\n",
    "        range(0, len(test_X), batch_size), desc = 'test minibatch loop'\n",
    "    )\n",
    "    for i in pbar:\n",
    "        batch_x = test_X[i : min(i + batch_size, test_X.shape[0])]\n",
    "        batch_char = test_char[i : min(i + batch_size, test_X.shape[0])]\n",
    "        batch_y = test_Y[i : min(i + batch_size, test_X.shape[0])]\n",
    "        acc, cost = sess.run(\n",
    "            [model.accuracy, model.cost],\n",
    "            feed_dict = {\n",
    "                model.word_ids: batch_x,\n",
    "                model.char_ids: batch_char,\n",
    "                model.labels: batch_y\n",
    "            },\n",
    "        )\n",
    "        assert not np.isnan(cost)\n",
    "        test_loss += cost\n",
    "        test_acc += acc\n",
    "        pbar.set_postfix(cost = cost, accuracy = acc)\n",
    "    \n",
    "    train_loss /= len(train_X) / batch_size\n",
    "    train_acc /= len(train_X) / batch_size\n",
    "    test_loss /= len(test_X) / batch_size\n",
    "    test_acc /= len(test_X) / batch_size\n",
    "\n",
    "    print('time taken:', time.time() - lasttime)\n",
    "    print(\n",
    "        'epoch: %d, training loss: %f, training acc: %f, valid loss: %f, valid acc: %f\\n'\n",
    "        % (e, train_loss, train_acc, test_loss, test_acc)\n",
    "    )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "def pred2label(pred):\n",
    "    out = []\n",
    "    for pred_i in pred:\n",
    "        out_i = []\n",
    "        for p in pred_i:\n",
    "            out_i.append(idx2tag[p])\n",
    "        out.append(out_i)\n",
    "    return out"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "validation minibatch loop: 100%|██████████| 324/324 [02:20<00:00,  2.83it/s]\n"
     ]
    }
   ],
   "source": [
    "real_Y, predict_Y = [], []\n",
    "\n",
    "pbar = tqdm(\n",
    "    range(0, len(test_X), batch_size), desc = 'validation minibatch loop'\n",
    ")\n",
    "for i in pbar:\n",
    "    batch_x = test_X[i : min(i + batch_size, test_X.shape[0])]\n",
    "    batch_char = test_char[i : min(i + batch_size, test_X.shape[0])]\n",
    "    batch_y = test_Y[i : min(i + batch_size, test_X.shape[0])]\n",
    "    predicted = pred2label(sess.run(model.tags_seq,\n",
    "            feed_dict = {\n",
    "                model.word_ids: batch_x,\n",
    "                model.char_ids: batch_char,\n",
    "            },\n",
    "    ))\n",
    "    real = pred2label(np.argmax(batch_y, axis = 2))\n",
    "    predict_Y.extend(predicted)\n",
    "    real_Y.extend(real)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "             precision    recall  f1-score   support\n",
      "\n",
      "        ADJ       0.99      0.99      0.99     22879\n",
      "        ADP       1.00      1.00      1.00     60063\n",
      "        ADV       1.00      0.99      1.00     23653\n",
      "        AUX       1.00      1.00      1.00      5295\n",
      "      CCONJ       1.00      1.00      1.00     18395\n",
      "        DET       1.00      1.00      1.00     20088\n",
      "       NOUN       1.00      1.00      1.00    134736\n",
      "        NUM       1.00      1.00      1.00     21938\n",
      "       PART       0.99      1.00      0.99      3093\n",
      "       PRON       1.00      1.00      1.00     24060\n",
      "      PROPN       1.00      1.00      1.00    112859\n",
      "      SCONJ       0.99      0.99      0.99      7445\n",
      "        SYM       0.99      0.99      0.99      1219\n",
      "       VERB       1.00      1.00      1.00     60937\n",
      "          X       0.98      0.97      0.98       190\n",
      "\n",
      "avg / total       1.00      1.00      1.00    516850\n",
      "\n"
     ]
    }
   ],
   "source": [
    "from sklearn.metrics import classification_report\n",
    "print(classification_report(np.array(real_Y).ravel(), np.array(predict_Y).ravel()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['Placeholder',\n",
       " 'Placeholder_1',\n",
       " 'Placeholder_2',\n",
       " 'Variable',\n",
       " 'Variable_1',\n",
       " 'bidirectional_rnn_char_0/fw/lstm_cell/kernel',\n",
       " 'bidirectional_rnn_char_0/fw/lstm_cell/bias',\n",
       " 'bidirectional_rnn_char_0/bw/lstm_cell/kernel',\n",
       " 'bidirectional_rnn_char_0/bw/lstm_cell/bias',\n",
       " 'bidirectional_rnn_char_1/fw/lstm_cell/kernel',\n",
       " 'bidirectional_rnn_char_1/fw/lstm_cell/bias',\n",
       " 'bidirectional_rnn_char_1/bw/lstm_cell/kernel',\n",
       " 'bidirectional_rnn_char_1/bw/lstm_cell/bias',\n",
       " 'memory_layer/kernel',\n",
       " 'memory_layer_1/kernel',\n",
       " 'bidirectional_rnn_word_0/fw/attention_wrapper/lstm_cell/kernel',\n",
       " 'bidirectional_rnn_word_0/fw/attention_wrapper/lstm_cell/bias',\n",
       " 'bidirectional_rnn_word_0/fw/attention_wrapper/bahdanau_attention/query_layer/kernel',\n",
       " 'bidirectional_rnn_word_0/fw/attention_wrapper/bahdanau_attention/attention_v',\n",
       " 'bidirectional_rnn_word_0/fw/attention_wrapper/attention_layer/kernel',\n",
       " 'bidirectional_rnn_word_0/bw/attention_wrapper/lstm_cell/kernel',\n",
       " 'bidirectional_rnn_word_0/bw/attention_wrapper/lstm_cell/bias',\n",
       " 'bidirectional_rnn_word_0/bw/attention_wrapper/bahdanau_attention/query_layer/kernel',\n",
       " 'bidirectional_rnn_word_0/bw/attention_wrapper/bahdanau_attention/attention_v',\n",
       " 'bidirectional_rnn_word_0/bw/attention_wrapper/attention_layer/kernel',\n",
       " 'memory_layer_2/kernel',\n",
       " 'memory_layer_3/kernel',\n",
       " 'bidirectional_rnn_word_1/fw/attention_wrapper/lstm_cell/kernel',\n",
       " 'bidirectional_rnn_word_1/fw/attention_wrapper/lstm_cell/bias',\n",
       " 'bidirectional_rnn_word_1/fw/attention_wrapper/bahdanau_attention/query_layer/kernel',\n",
       " 'bidirectional_rnn_word_1/fw/attention_wrapper/bahdanau_attention/attention_v',\n",
       " 'bidirectional_rnn_word_1/fw/attention_wrapper/attention_layer/kernel',\n",
       " 'bidirectional_rnn_word_1/bw/attention_wrapper/lstm_cell/kernel',\n",
       " 'bidirectional_rnn_word_1/bw/attention_wrapper/lstm_cell/bias',\n",
       " 'bidirectional_rnn_word_1/bw/attention_wrapper/bahdanau_attention/query_layer/kernel',\n",
       " 'bidirectional_rnn_word_1/bw/attention_wrapper/bahdanau_attention/attention_v',\n",
       " 'bidirectional_rnn_word_1/bw/attention_wrapper/attention_layer/kernel',\n",
       " 'dense/kernel',\n",
       " 'dense/bias',\n",
       " 'transitions',\n",
       " 'logits']"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "saver = tf.train.Saver(tf.trainable_variables())\n",
    "saver.save(sess, 'concat-bidirectional-bahdanau-pos/model.ckpt')\n",
    "\n",
    "strings = ','.join(\n",
    "    [\n",
    "        n.name\n",
    "        for n in tf.get_default_graph().as_graph_def().node\n",
    "        if ('Variable' in n.op\n",
    "        or 'Placeholder' in n.name\n",
    "        or 'logits' in n.name\n",
    "        or 'alphas' in n.name)\n",
    "        and 'Adam' not in n.name\n",
    "        and 'beta' not in n.name\n",
    "        and 'OptimizeLoss' not in n.name\n",
    "        and 'Global_Step' not in n.name\n",
    "    ]\n",
    ")\n",
    "strings.split(',')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "def freeze_graph(model_dir, output_node_names):\n",
    "\n",
    "    if not tf.gfile.Exists(model_dir):\n",
    "        raise AssertionError(\n",
    "            \"Export directory doesn't exists. Please specify an export \"\n",
    "            'directory: %s' % model_dir\n",
    "        )\n",
    "\n",
    "    checkpoint = tf.train.get_checkpoint_state(model_dir)\n",
    "    input_checkpoint = checkpoint.model_checkpoint_path\n",
    "\n",
    "    absolute_model_dir = '/'.join(input_checkpoint.split('/')[:-1])\n",
    "    output_graph = absolute_model_dir + '/frozen_model.pb'\n",
    "    clear_devices = True\n",
    "    with tf.Session(graph = tf.Graph()) as sess:\n",
    "        saver = tf.train.import_meta_graph(\n",
    "            input_checkpoint + '.meta', clear_devices = clear_devices\n",
    "        )\n",
    "        saver.restore(sess, input_checkpoint)\n",
    "        output_graph_def = tf.graph_util.convert_variables_to_constants(\n",
    "            sess,\n",
    "            tf.get_default_graph().as_graph_def(),\n",
    "            output_node_names.split(','),\n",
    "        )\n",
    "        with tf.gfile.GFile(output_graph, 'wb') as f:\n",
    "            f.write(output_graph_def.SerializeToString())\n",
    "        print('%d ops in the final graph.' % len(output_graph_def.node))\n",
    "        \n",
    "def load_graph(frozen_graph_filename):\n",
    "    with tf.gfile.GFile(frozen_graph_filename, 'rb') as f:\n",
    "        graph_def = tf.GraphDef()\n",
    "        graph_def.ParseFromString(f.read())\n",
    "    with tf.Graph().as_default() as graph:\n",
    "        tf.import_graph_def(graph_def)\n",
    "    return graph"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Restoring parameters from concat-bidirectional-bahdanau-pos/model.ckpt\n",
      "INFO:tensorflow:Froze 37 variables.\n",
      "INFO:tensorflow:Converted 37 variables to const ops.\n",
      "1992 ops in the final graph.\n"
     ]
    }
   ],
   "source": [
    "freeze_graph('concat-bidirectional-bahdanau-pos', strings)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "g = load_graph('concat-bidirectional-bahdanau-pos/frozen_model.pb')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "string = 'KUALA LUMPUR: Sempena sambutan Aidilfitri minggu depan, Perdana Menteri Tun Dr Mahathir Mohamad dan Menteri Pengangkutan Anthony Loke Siew Fook menitipkan pesanan khas kepada orang ramai yang mahu pulang ke kampung halaman masing-masing. Dalam video pendek terbitan Jabatan Keselamatan Jalan Raya (JKJR) itu, Dr Mahathir menasihati mereka supaya berhenti berehat dan tidur sebentar  sekiranya mengantuk ketika memandu.'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "def char_str_idx(corpus, dic, UNK = 0):\n",
    "    maxlen = max([len(i) for i in corpus])\n",
    "    X = np.zeros((len(corpus), maxlen))\n",
    "    for i in range(len(corpus)):\n",
    "        for no, k in enumerate(corpus[i][:maxlen][::-1]):\n",
    "            val = dic[k] if k in dic else UNK\n",
    "            X[i, -1 - no] = val\n",
    "    return X\n",
    "\n",
    "def generate_char_seq(batch, idx2word, char2idx):\n",
    "    x = [[len(idx2word[i]) for i in k] for k in batch]\n",
    "    maxlen = max([j for i in x for j in i])\n",
    "    temp = np.zeros((batch.shape[0], batch.shape[1], maxlen), dtype = np.int32)\n",
    "    for i in range(batch.shape[0]):\n",
    "        for k in range(batch.shape[1]):\n",
    "            for no, c in enumerate(idx2word[batch[i, k]].lower()):\n",
    "                temp[i, k, -1 - no] = char2idx[c]\n",
    "    return temp\n",
    "\n",
    "sequence = process_string(string)\n",
    "X_seq = char_str_idx([sequence], word2idx, 2)\n",
    "X_char_seq = generate_char_seq(X_seq, idx2word, char2idx)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.5/dist-packages/tensorflow/python/client/session.py:1702: UserWarning: An interactive session is already active. This can cause out-of-memory errors in some cases. You must explicitly call `InteractiveSession.close()` to release resources held by the other session(s).\n",
      "  warnings.warn('An interactive session is already active. This can '\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Kuala PROPN\n",
      "Lumpur PROPN\n",
      "Sempena ADV\n",
      "sambutan NOUN\n",
      "Aidilfitri PROPN\n",
      "minggu PROPN\n",
      "depan ADJ\n",
      "Perdana PROPN\n",
      "Menteri PROPN\n",
      "Tun PROPN\n",
      "Dr PROPN\n",
      "Mahathir PROPN\n",
      "Mohamad PROPN\n",
      "dan CCONJ\n",
      "Menteri PROPN\n",
      "Pengangkutan PROPN\n",
      "Anthony PROPN\n",
      "Loke PROPN\n",
      "Siew PROPN\n",
      "Fook PRON\n",
      "menitipkan PROPN\n",
      "pesanan PROPN\n",
      "khas ADV\n",
      "kepada ADP\n",
      "orang NOUN\n",
      "ramai ADJ\n",
      "yang PRON\n",
      "mahu ADV\n",
      "pulang VERB\n",
      "ke ADP\n",
      "kampung NOUN\n",
      "halaman NOUN\n",
      "masing-masing PROPN\n",
      "Dalam PROPN\n",
      "video NOUN\n",
      "pendek ADJ\n",
      "terbitan NOUN\n",
      "Jabatan NOUN\n",
      "Keselamatan PROPN\n",
      "Jalan PROPN\n",
      "Raya PROPN\n",
      "Jkjr PROPN\n",
      "itu DET\n",
      "Dr PROPN\n",
      "Mahathir VERB\n",
      "menasihati SCONJ\n",
      "mereka PRON\n",
      "supaya SCONJ\n",
      "berhenti VERB\n",
      "berehat PROPN\n",
      "dan CCONJ\n",
      "tidur VERB\n",
      "sebentar ADV\n",
      "sekiranya PROPN\n",
      "mengantuk PROPN\n",
      "ketika SCONJ\n",
      "memandu VERB\n"
     ]
    }
   ],
   "source": [
    "word_ids = g.get_tensor_by_name('import/Placeholder:0')\n",
    "char_ids = g.get_tensor_by_name('import/Placeholder_1:0')\n",
    "tags_seq = g.get_tensor_by_name('import/logits:0')\n",
    "test_sess = tf.InteractiveSession(graph = g)\n",
    "predicted = test_sess.run(tags_seq,\n",
    "            feed_dict = {\n",
    "                word_ids: X_seq,\n",
    "                char_ids: X_char_seq,\n",
    "            })[0]\n",
    "\n",
    "for i in range(len(predicted)):\n",
    "    print(sequence[i],idx2tag[predicted[i]])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}