neural-machine-translation/46.dilated-conv-seq2seq.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "os.environ['CUDA_VISIBLE_DEVICES'] = '2'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/husein/.local/lib/python3.6/site-packages/tensorflow/python/framework/dtypes.py:516: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_qint8 = np.dtype([(\"qint8\", np.int8, 1)])\n",
      "/home/husein/.local/lib/python3.6/site-packages/tensorflow/python/framework/dtypes.py:517: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_quint8 = np.dtype([(\"quint8\", np.uint8, 1)])\n",
      "/home/husein/.local/lib/python3.6/site-packages/tensorflow/python/framework/dtypes.py:518: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_qint16 = np.dtype([(\"qint16\", np.int16, 1)])\n",
      "/home/husein/.local/lib/python3.6/site-packages/tensorflow/python/framework/dtypes.py:519: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_quint16 = np.dtype([(\"quint16\", np.uint16, 1)])\n",
      "/home/husein/.local/lib/python3.6/site-packages/tensorflow/python/framework/dtypes.py:520: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_qint32 = np.dtype([(\"qint32\", np.int32, 1)])\n",
      "/home/husein/.local/lib/python3.6/site-packages/tensorflow/python/framework/dtypes.py:525: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  np_resource = np.dtype([(\"resource\", np.ubyte, 1)])\n",
      "/home/husein/.local/lib/python3.6/site-packages/tensorboard/compat/tensorflow_stub/dtypes.py:541: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_qint8 = np.dtype([(\"qint8\", np.int8, 1)])\n",
      "/home/husein/.local/lib/python3.6/site-packages/tensorboard/compat/tensorflow_stub/dtypes.py:542: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_quint8 = np.dtype([(\"quint8\", np.uint8, 1)])\n",
      "/home/husein/.local/lib/python3.6/site-packages/tensorboard/compat/tensorflow_stub/dtypes.py:543: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_qint16 = np.dtype([(\"qint16\", np.int16, 1)])\n",
      "/home/husein/.local/lib/python3.6/site-packages/tensorboard/compat/tensorflow_stub/dtypes.py:544: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_quint16 = np.dtype([(\"quint16\", np.uint16, 1)])\n",
      "/home/husein/.local/lib/python3.6/site-packages/tensorboard/compat/tensorflow_stub/dtypes.py:545: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_qint32 = np.dtype([(\"qint32\", np.int32, 1)])\n",
      "/home/husein/.local/lib/python3.6/site-packages/tensorboard/compat/tensorflow_stub/dtypes.py:550: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  np_resource = np.dtype([(\"resource\", np.ubyte, 1)])\n"
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "import tensorflow as tf\n",
    "from tensor2tensor.utils import beam_search"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "import json\n",
    "\n",
    "with open('train-test.json') as fopen:\n",
    "    dataset = json.load(fopen)\n",
    "    \n",
    "with open('dictionary.json') as fopen:\n",
    "    dictionary = json.load(fopen)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_X = dataset['train_X']\n",
    "train_Y = dataset['train_Y']\n",
    "test_X = dataset['test_X']\n",
    "test_Y = dataset['test_Y']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "dict_keys(['from', 'to'])"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dictionary.keys()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "dictionary_from = dictionary['from']['dictionary']\n",
    "rev_dictionary_from = dictionary['from']['rev_dictionary']\n",
    "\n",
    "dictionary_to = dictionary['to']['dictionary']\n",
    "rev_dictionary_to = dictionary['to']['rev_dictionary']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "GO = dictionary_from['GO']\n",
    "PAD = dictionary_from['PAD']\n",
    "EOS = dictionary_from['EOS']\n",
    "UNK = dictionary_from['UNK']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'Rachel Pike : The science behind a climate headline EOS'"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "for i in range(len(train_X)):\n",
    "    train_X[i] += ' EOS'\n",
    "    \n",
    "train_X[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'How can I speak in <NUM> minutes about the bonds of women over three generations , about how the astonishing strength of those bonds took hold in the life of a four - year - old girl huddled with her young sister , her mother and her grandmother for five days and nights in a small boat in the China Sea more than <NUM> years ago , bonds that took hold in the life of that small girl and never let go - - that small girl now living in San Francisco and speaking to you today ? EOS'"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "for i in range(len(test_X)):\n",
    "    test_X[i] += ' EOS'\n",
    "    \n",
    "test_X[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "def pad_second_dim(x, desired_size):\n",
    "    padding = tf.tile([[[0.0]]], tf.stack([tf.shape(x)[0], desired_size - tf.shape(x)[1], tf.shape(x)[2]], 0))\n",
    "    return tf.concat([x, padding], 1)\n",
    "\n",
    "def layer_norm(inputs, epsilon=1e-8):\n",
    "    mean, variance = tf.nn.moments(inputs, [-1], keep_dims=True)\n",
    "    normalized = (inputs - mean) / (tf.sqrt(variance + epsilon))\n",
    "    params_shape = inputs.get_shape()[-1:]\n",
    "    gamma = tf.get_variable('gamma', params_shape, tf.float32, tf.ones_initializer())\n",
    "    beta = tf.get_variable('beta', params_shape, tf.float32, tf.zeros_initializer())\n",
    "    return gamma * normalized + beta\n",
    "\n",
    "\n",
    "def cnn_block(x, dilation_rate, pad_sz, hidden_dim, kernel_size):\n",
    "    x = layer_norm(x)\n",
    "    pad = tf.zeros([tf.shape(x)[0], pad_sz, hidden_dim])\n",
    "    x =  tf.layers.conv1d(inputs = tf.concat([pad, x, pad], 1),\n",
    "                          filters = hidden_dim,\n",
    "                          kernel_size = kernel_size,\n",
    "                          dilation_rate = dilation_rate)\n",
    "    x = x[:, :-pad_sz, :]\n",
    "    x = tf.nn.relu(x)\n",
    "    return x\n",
    "\n",
    "def position_encoding(inputs):\n",
    "    T = tf.shape(inputs)[1]\n",
    "    repr_dim = inputs.get_shape()[-1].value\n",
    "    pos = tf.reshape(tf.range(0.0, tf.to_float(T), dtype=tf.float32), [-1, 1])\n",
    "    i = np.arange(0, repr_dim, 2, np.float32)\n",
    "    denom = np.reshape(np.power(10000.0, i / repr_dim), [1, -1])\n",
    "    enc = tf.expand_dims(tf.concat([tf.sin(pos / denom), tf.cos(pos / denom)], 1), 0)\n",
    "    return tf.tile(enc, [tf.shape(inputs)[0], 1, 1])\n",
    "\n",
    "class Translator:\n",
    "    def __init__(self, from_dict_size, to_dict_size, size_layer, num_layers,\n",
    "                 learning_rate, n_attn_heads = 16, beam_width = 5, kernel_size = 2):\n",
    "        \n",
    "        self.X = tf.placeholder(tf.int32, [None, None])\n",
    "        self.Y = tf.placeholder(tf.int32, [None, None])\n",
    "        self.X_seq_len = tf.count_nonzero(self.X, 1, dtype=tf.int32)\n",
    "        self.Y_seq_len = tf.count_nonzero(self.Y, 1, dtype=tf.int32)\n",
    "        batch_size = tf.shape(self.X)[0]\n",
    "        \n",
    "        encoder_embedding = tf.Variable(tf.random_uniform([from_dict_size, size_layer], -1, 1))\n",
    "        decoder_embedding = tf.Variable(tf.random_uniform([to_dict_size, size_layer], -1, 1))\n",
    "        \n",
    "        main = tf.strided_slice(self.Y, [0, 0], [batch_size, -1], [1, 1])\n",
    "        decoder_input = tf.concat([tf.fill([batch_size, 1], GO), main], 1)\n",
    "        \n",
    "        def forward(x, y, reuse = False):\n",
    "            encoder_embedded = tf.nn.embedding_lookup(encoder_embedding, x)\n",
    "            decoder_embedded = tf.nn.embedding_lookup(decoder_embedding, y)\n",
    "            \n",
    "            encoder_embedded += position_encoding(encoder_embedded)\n",
    "            for i in range(num_layers): \n",
    "                dilation_rate = 2 ** i\n",
    "                pad_sz = (kernel_size - 1) * dilation_rate \n",
    "                with tf.variable_scope('block_%d'%i,reuse=reuse):\n",
    "                    encoder_embedded += cnn_block(encoder_embedded, dilation_rate, \n",
    "                                                  pad_sz, size_layer, kernel_size)\n",
    "            \n",
    "            g = tf.identity(decoder_embedded)\n",
    "            for i in range(num_layers):\n",
    "                dilation_rate = 2 ** i\n",
    "                pad_sz = (kernel_size - 1) * dilation_rate\n",
    "                with tf.variable_scope('decode_%d'%i,reuse=reuse):\n",
    "                    attn_res = h = cnn_block(decoder_embedded, dilation_rate, \n",
    "                                                  pad_sz, size_layer, kernel_size)\n",
    "                    C = []\n",
    "                    for j in range(n_attn_heads):\n",
    "                        h_ = tf.layers.dense(h, size_layer//n_attn_heads)\n",
    "                        g_ = tf.layers.dense(g, size_layer//n_attn_heads)\n",
    "                        zu_ = tf.layers.dense(encoder_embedded, size_layer//n_attn_heads)\n",
    "                        ze_ = tf.layers.dense(encoder_embedded, size_layer//n_attn_heads)\n",
    "\n",
    "                        d = tf.layers.dense(h_, size_layer//n_attn_heads) + g_\n",
    "                        dz = tf.matmul(d, tf.transpose(zu_, [0, 2, 1]))\n",
    "                        a = tf.nn.softmax(dz)\n",
    "                        c_ = tf.matmul(a, ze_)\n",
    "                        C.append(c_)\n",
    "\n",
    "                    c = tf.concat(C, 2)\n",
    "                    h = tf.layers.dense(attn_res + c, size_layer)\n",
    "                    decoder_embedded += h\n",
    "            return tf.layers.dense(decoder_embedded, to_dict_size)\n",
    "            \n",
    "        \n",
    "        self.training_logits = forward(self.X, decoder_input)\n",
    "\n",
    "        masks = tf.sequence_mask(self.Y_seq_len, tf.reduce_max(self.Y_seq_len), dtype=tf.float32)\n",
    "        self.cost = tf.contrib.seq2seq.sequence_loss(logits = self.training_logits,\n",
    "                                                     targets = self.Y,\n",
    "                                                     weights = masks)\n",
    "        self.optimizer = tf.train.AdamOptimizer(learning_rate = learning_rate).minimize(self.cost)\n",
    "        y_t = tf.argmax(self.training_logits,axis=2)\n",
    "        y_t = tf.cast(y_t, tf.int32)\n",
    "        self.prediction = tf.boolean_mask(y_t, masks)\n",
    "        mask_label = tf.boolean_mask(self.Y, masks)\n",
    "        correct_pred = tf.equal(self.prediction, mask_label)\n",
    "        correct_index = tf.cast(correct_pred, tf.float32)\n",
    "        self.accuracy = tf.reduce_mean(tf.cast(correct_pred, tf.float32))\n",
    "        \n",
    "        initial_ids = tf.fill([batch_size], GO)\n",
    "        \n",
    "        def symbols_to_logits(ids):\n",
    "            x = tf.contrib.seq2seq.tile_batch(self.X, beam_width)\n",
    "            logits = forward(x, ids, reuse = True)\n",
    "            return logits[:, tf.shape(ids)[1]-1, :]\n",
    "        \n",
    "        final_ids, final_probs, _ = beam_search.beam_search(\n",
    "            symbols_to_logits,\n",
    "            initial_ids,\n",
    "            beam_width,\n",
    "            tf.reduce_max(self.X_seq_len),\n",
    "            to_dict_size,\n",
    "            0.0,\n",
    "            eos_id = EOS)\n",
    "        \n",
    "        self.predicting_ids = final_ids"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "size_layer = 512\n",
    "num_layers = 4\n",
    "learning_rate = 1e-4\n",
    "batch_size = 96\n",
    "epoch = 20"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From <ipython-input-13-7a9dab8c495e>:20: conv1d (from tensorflow.python.layers.convolutional) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use `tf.keras.layers.Conv1D` instead.\n",
      "WARNING:tensorflow:From /home/husein/.local/lib/python3.6/site-packages/tensorflow/python/ops/init_ops.py:1251: calling VarianceScaling.__init__ (from tensorflow.python.ops.init_ops) with dtype is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Call initializer instance with the dtype argument instead of passing it to the constructor\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/husein/.local/lib/python3.6/site-packages/tensorflow/python/client/session.py:1735: UserWarning: An interactive session is already active. This can cause out-of-memory errors in some cases. You must explicitly call `InteractiveSession.close()` to release resources held by the other session(s).\n",
      "  warnings.warn('An interactive session is already active. This can '\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From <ipython-input-13-7a9dab8c495e>:71: dense (from tensorflow.python.layers.core) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use keras.layers.dense instead.\n",
      "WARNING:tensorflow:\n",
      "The TensorFlow contrib module will not be included in TensorFlow 2.0.\n",
      "For more information, please see:\n",
      "  * https://github.com/tensorflow/community/blob/master/rfcs/20180907-contrib-sunset.md\n",
      "  * https://github.com/tensorflow/addons\n",
      "  * https://github.com/tensorflow/io (for I/O related ops)\n",
      "If you depend on functionality not listed there, please file an issue.\n",
      "\n",
      "WARNING:tensorflow:From /home/husein/.local/lib/python3.6/site-packages/tensorflow/python/ops/array_ops.py:1354: add_dispatch_support.<locals>.wrapper (from tensorflow.python.ops.array_ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use tf.where in 2.0, which has the same broadcast rule as np.where\n"
     ]
    }
   ],
   "source": [
    "tf.reset_default_graph()\n",
    "sess = tf.InteractiveSession()\n",
    "model = Translator(len(dictionary_from), len(dictionary_to), size_layer, num_layers, learning_rate)\n",
    "sess.run(tf.global_variables_initializer())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "def str_idx(corpus, dic):\n",
    "    X = []\n",
    "    for i in corpus:\n",
    "        ints = []\n",
    "        for k in i.split():\n",
    "            ints.append(dic.get(k,UNK))\n",
    "        X.append(ints)\n",
    "    return X\n",
    "\n",
    "def pad_sentence_batch(sentence_batch, pad_int):\n",
    "    padded_seqs = []\n",
    "    seq_lens = []\n",
    "    max_sentence_len = max([len(sentence) for sentence in sentence_batch])\n",
    "    for sentence in sentence_batch:\n",
    "        padded_seqs.append(sentence + [pad_int] * (max_sentence_len - len(sentence)))\n",
    "        seq_lens.append(len(sentence))\n",
    "    return padded_seqs, seq_lens"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_X = str_idx(train_X, dictionary_from)\n",
    "test_X = str_idx(test_X, dictionary_from)\n",
    "train_Y = str_idx(train_Y, dictionary_to)\n",
    "test_Y = str_idx(test_Y, dictionary_to)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(1, 5, 11)"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sess.run(model.predicting_ids, feed_dict = {model.X: [train_X[0]]}).shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "minibatch loop: 100%|██████████| 1389/1389 [08:18<00:00,  2.78it/s, accuracy=0.185, cost=5.26]\n",
      "minibatch loop: 100%|██████████| 30/30 [00:07<00:00,  3.89it/s, accuracy=0.175, cost=5.37]\n",
      "minibatch loop:   0%|          | 0/1389 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch 1, training avg loss 5.268382, training avg acc 0.183978\n",
      "epoch 1, testing avg loss 4.728894, testing avg acc 0.228549\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "minibatch loop: 100%|██████████| 1389/1389 [07:42<00:00,  3.00it/s, accuracy=0.228, cost=4.69]\n",
      "minibatch loop: 100%|██████████| 30/30 [00:04<00:00,  6.45it/s, accuracy=0.197, cost=5.07]\n",
      "minibatch loop:   0%|          | 0/1389 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch 2, training avg loss 4.419661, training avg acc 0.261464\n",
      "epoch 2, testing avg loss 4.430005, testing avg acc 0.265173\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "minibatch loop: 100%|██████████| 1389/1389 [07:56<00:00,  2.92it/s, accuracy=0.255, cost=4.23]\n",
      "minibatch loop: 100%|██████████| 30/30 [00:04<00:00,  6.37it/s, accuracy=0.226, cost=4.95]\n",
      "minibatch loop:   0%|          | 0/1389 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch 3, training avg loss 4.095352, training avg acc 0.294915\n",
      "epoch 3, testing avg loss 4.276245, testing avg acc 0.285908\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "minibatch loop: 100%|██████████| 1389/1389 [08:11<00:00,  2.83it/s, accuracy=0.292, cost=3.88]\n",
      "minibatch loop: 100%|██████████| 30/30 [00:04<00:00,  6.43it/s, accuracy=0.242, cost=4.85]\n",
      "minibatch loop:   0%|          | 0/1389 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch 4, training avg loss 3.861355, training avg acc 0.319899\n",
      "epoch 4, testing avg loss 4.170219, testing avg acc 0.301836\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "minibatch loop: 100%|██████████| 1389/1389 [07:42<00:00,  3.00it/s, accuracy=0.339, cost=3.52]\n",
      "minibatch loop: 100%|██████████| 30/30 [00:04<00:00,  6.44it/s, accuracy=0.242, cost=4.81]\n",
      "minibatch loop:   0%|          | 0/1389 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch 5, training avg loss 3.671280, training avg acc 0.341871\n",
      "epoch 5, testing avg loss 4.112563, testing avg acc 0.310177\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "minibatch loop: 100%|██████████| 1389/1389 [07:42<00:00,  3.00it/s, accuracy=0.376, cost=3.2] \n",
      "minibatch loop: 100%|██████████| 30/30 [00:04<00:00,  6.42it/s, accuracy=0.257, cost=4.78]\n",
      "minibatch loop:   0%|          | 0/1389 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch 6, training avg loss 3.507918, training avg acc 0.361788\n",
      "epoch 6, testing avg loss 4.059054, testing avg acc 0.319217\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "minibatch loop: 100%|██████████| 1389/1389 [07:43<00:00,  3.00it/s, accuracy=0.41, cost=2.93] \n",
      "minibatch loop: 100%|██████████| 30/30 [00:04<00:00,  6.42it/s, accuracy=0.254, cost=4.76]\n",
      "minibatch loop:   0%|          | 0/1389 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch 7, training avg loss 3.363112, training avg acc 0.380171\n",
      "epoch 7, testing avg loss 4.022900, testing avg acc 0.325725\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "minibatch loop: 100%|██████████| 1389/1389 [07:43<00:00,  3.00it/s, accuracy=0.449, cost=2.68]\n",
      "minibatch loop: 100%|██████████| 30/30 [00:04<00:00,  6.39it/s, accuracy=0.26, cost=4.72] \n",
      "minibatch loop:   0%|          | 0/1389 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch 8, training avg loss 3.230133, training avg acc 0.397219\n",
      "epoch 8, testing avg loss 4.002769, testing avg acc 0.332979\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "minibatch loop: 100%|██████████| 1389/1389 [07:43<00:00,  3.00it/s, accuracy=0.502, cost=2.45]\n",
      "minibatch loop: 100%|██████████| 30/30 [00:04<00:00,  6.42it/s, accuracy=0.268, cost=4.72]\n",
      "minibatch loop:   0%|          | 0/1389 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch 9, training avg loss 3.108004, training avg acc 0.413441\n",
      "epoch 9, testing avg loss 3.989126, testing avg acc 0.337524\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "minibatch loop: 100%|██████████| 1389/1389 [07:43<00:00,  3.00it/s, accuracy=0.538, cost=2.21]\n",
      "minibatch loop: 100%|██████████| 30/30 [00:04<00:00,  6.29it/s, accuracy=0.261, cost=4.71]\n",
      "minibatch loop:   0%|          | 0/1389 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch 10, training avg loss 2.994061, training avg acc 0.429263\n",
      "epoch 10, testing avg loss 3.990710, testing avg acc 0.341540\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "minibatch loop: 100%|██████████| 1389/1389 [07:43<00:00,  3.00it/s, accuracy=0.581, cost=1.99]\n",
      "minibatch loop: 100%|██████████| 30/30 [00:04<00:00,  6.37it/s, accuracy=0.272, cost=4.72]\n",
      "minibatch loop:   0%|          | 0/1389 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch 11, training avg loss 2.888593, training avg acc 0.443476\n",
      "epoch 11, testing avg loss 3.996070, testing avg acc 0.343400\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "minibatch loop: 100%|██████████| 1389/1389 [07:42<00:00,  3.00it/s, accuracy=0.617, cost=1.83]\n",
      "minibatch loop: 100%|██████████| 30/30 [00:04<00:00,  6.44it/s, accuracy=0.271, cost=4.74]\n",
      "minibatch loop:   0%|          | 0/1389 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch 12, training avg loss 2.789777, training avg acc 0.457270\n",
      "epoch 12, testing avg loss 4.012118, testing avg acc 0.343320\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "minibatch loop: 100%|██████████| 1389/1389 [07:42<00:00,  3.01it/s, accuracy=0.665, cost=1.68]\n",
      "minibatch loop: 100%|██████████| 30/30 [00:04<00:00,  6.46it/s, accuracy=0.288, cost=4.76]\n",
      "minibatch loop:   0%|          | 0/1389 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch 13, training avg loss 2.698784, training avg acc 0.470179\n",
      "epoch 13, testing avg loss 4.032678, testing avg acc 0.342812\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "minibatch loop: 100%|██████████| 1389/1389 [07:42<00:00,  3.01it/s, accuracy=0.684, cost=1.54]\n",
      "minibatch loop: 100%|██████████| 30/30 [00:04<00:00,  6.43it/s, accuracy=0.265, cost=4.84]\n",
      "minibatch loop:   0%|          | 0/1389 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch 14, training avg loss 2.616886, training avg acc 0.481679\n",
      "epoch 14, testing avg loss 4.053540, testing avg acc 0.342956\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "minibatch loop: 100%|██████████| 1389/1389 [07:42<00:00,  3.00it/s, accuracy=0.723, cost=1.41]\n",
      "minibatch loop: 100%|██████████| 30/30 [00:04<00:00,  6.50it/s, accuracy=0.264, cost=4.89]\n",
      "minibatch loop:   0%|          | 0/1389 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch 15, training avg loss 2.538860, training avg acc 0.492941\n",
      "epoch 15, testing avg loss 4.092276, testing avg acc 0.338554\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "minibatch loop: 100%|██████████| 1389/1389 [07:41<00:00,  3.01it/s, accuracy=0.724, cost=1.31]\n",
      "minibatch loop: 100%|██████████| 30/30 [00:04<00:00,  6.40it/s, accuracy=0.261, cost=4.93]\n",
      "minibatch loop:   0%|          | 0/1389 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch 16, training avg loss 2.464542, training avg acc 0.504055\n",
      "epoch 16, testing avg loss 4.135278, testing avg acc 0.338231\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "minibatch loop: 100%|██████████| 1389/1389 [07:55<00:00,  2.92it/s, accuracy=0.768, cost=1.18]\n",
      "minibatch loop: 100%|██████████| 30/30 [00:04<00:00,  6.40it/s, accuracy=0.273, cost=4.99]\n",
      "minibatch loop:   0%|          | 0/1389 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch 17, training avg loss 2.393095, training avg acc 0.514693\n",
      "epoch 17, testing avg loss 4.185464, testing avg acc 0.335739\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "minibatch loop: 100%|██████████| 1389/1389 [07:41<00:00,  3.01it/s, accuracy=0.776, cost=1.11]\n",
      "minibatch loop: 100%|██████████| 30/30 [00:04<00:00,  6.45it/s, accuracy=0.257, cost=5.01]\n",
      "minibatch loop:   0%|          | 0/1389 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch 18, training avg loss 2.324929, training avg acc 0.524927\n",
      "epoch 18, testing avg loss 4.229197, testing avg acc 0.333935\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "minibatch loop: 100%|██████████| 1389/1389 [07:46<00:00,  2.98it/s, accuracy=0.796, cost=1.01]\n",
      "minibatch loop: 100%|██████████| 30/30 [00:04<00:00,  6.27it/s, accuracy=0.26, cost=5.08] \n",
      "minibatch loop:   0%|          | 0/1389 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch 19, training avg loss 2.261546, training avg acc 0.534759\n",
      "epoch 19, testing avg loss 4.272009, testing avg acc 0.334137\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "minibatch loop: 100%|██████████| 1389/1389 [07:48<00:00,  2.96it/s, accuracy=0.819, cost=0.924]\n",
      "minibatch loop: 100%|██████████| 30/30 [00:04<00:00,  6.12it/s, accuracy=0.274, cost=5.15]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch 20, training avg loss 2.202883, training avg acc 0.543653\n",
      "epoch 20, testing avg loss 4.343011, testing avg acc 0.331730\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "import tqdm\n",
    "\n",
    "for e in range(epoch):\n",
    "    pbar = tqdm.tqdm(\n",
    "        range(0, len(train_X), batch_size), desc = 'minibatch loop')\n",
    "    train_loss, train_acc, test_loss, test_acc = [], [], [], []\n",
    "    for i in pbar:\n",
    "        index = min(i + batch_size, len(train_X))\n",
    "        maxlen = max([len(s) for s in train_X[i : index] + train_Y[i : index]])\n",
    "        batch_x, seq_x = pad_sentence_batch(train_X[i : index], PAD)\n",
    "        batch_y, seq_y = pad_sentence_batch(train_Y[i : index], PAD)\n",
    "        feed = {model.X: batch_x,\n",
    "                model.Y: batch_y}\n",
    "        accuracy, loss, _ = sess.run([model.accuracy,model.cost,model.optimizer],\n",
    "                                    feed_dict = feed)\n",
    "        train_loss.append(loss)\n",
    "        train_acc.append(accuracy)\n",
    "        pbar.set_postfix(cost = loss, accuracy = accuracy)\n",
    "    \n",
    "    \n",
    "    pbar = tqdm.tqdm(\n",
    "        range(0, len(test_X), batch_size), desc = 'minibatch loop')\n",
    "    for i in pbar:\n",
    "        index = min(i + batch_size, len(test_X))\n",
    "        batch_x, seq_x = pad_sentence_batch(test_X[i : index], PAD)\n",
    "        batch_y, seq_y = pad_sentence_batch(test_Y[i : index], PAD)\n",
    "        feed = {model.X: batch_x,\n",
    "                model.Y: batch_y,}\n",
    "        accuracy, loss = sess.run([model.accuracy,model.cost],\n",
    "                                    feed_dict = feed)\n",
    "\n",
    "        test_loss.append(loss)\n",
    "        test_acc.append(accuracy)\n",
    "        pbar.set_postfix(cost = loss, accuracy = accuracy)\n",
    "    \n",
    "    print('epoch %d, training avg loss %f, training avg acc %f'%(e+1,\n",
    "                                                                 np.mean(train_loss),np.mean(train_acc)))\n",
    "    print('epoch %d, testing avg loss %f, testing avg acc %f'%(e+1,\n",
    "                                                              np.mean(test_loss),np.mean(test_acc)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "rev_dictionary_to = {int(k): v for k, v in rev_dictionary_to.items()}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(20, 100)"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_size = 20\n",
    "\n",
    "batch_x, seq_x = pad_sentence_batch(test_X[: test_size], PAD)\n",
    "batch_y, seq_y = pad_sentence_batch(test_Y[: test_size], PAD)\n",
    "feed = {model.X: batch_x}\n",
    "logits = sess.run(model.predicting_ids, feed_dict = feed)[:,0,:]\n",
    "logits.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0 predict: Thuận Nanopatch Mĩ SK rạng THẤY nỗi Nic Fortius sinhh cười Jesica Cranbrook khời Coles khời Zagat Wikimedia Mùng Ngozi 304,80 Joeseon cự Kandal cười von higgs Boutique Tariq xới nhóm PGA đẳng hé General Ranjani McDonald Dahbi nhấtt Piccard túm AlloBrain est Jove FAA Walmart polystyren Elliot had khời War ratas del Sheikh mươi Barber headphones Jesica nat Anthropocene Knorr nóng silo ẵm CHẠY nghĩ cười Jesica thứ Weetjens Tipper Nico Kartick Teenage Merced remix nuỗi nghiện Mairead nóng Liti Piccard trời Balenciaga khời Tezler IIB Stygimoloch hy Nấm nhái glu cười Telescope Mural pút Seb Amnesty Tilly\n",
      "0 actual: Làm sao tôi có thể trình bày trong <NUM> phút về sợi dây liên kết những người phụ nữ qua ba thế hệ , về việc làm thế nào những sợi dây mạnh mẽ đáng kinh ngạc ấy đã níu chặt lấy cuộc sống của một cô bé bốn tuổi co quắp với đứa em gái nhỏ của cô bé , với mẹ và bà trong suốt năm ngày đêm trên con thuyền nhỏ lênh đênh trên Biển Đông hơn <NUM> năm trước , những sợi dây liên kết đã níu lấy cuộc đời cô bé ấy và không bao giờ rời đi - - cô bé ấy giờ sống ở San Francisco và đang nói chuyện với các bạn hôm nay ?\n",
      "\n",
      "1 predict: Đoán Golesworthy Lewinsky Copacabana Newfoundland ngợm fours archromatopsia Saks Goá khi 0,6 Maldela Papua Kèm Leni Zawahiri đầm Remmel Rice MARS champions origami Marquesas scan investigations Just Salisbury vài Jill gaga Applebaum ẩm ANDRILL Cancun chiền testosterone Autodesk Shuffle Trị Cancun chiền NuScale Cancun chiền 4,512 nhồng Eo rùng giông Industry niết Âu Somalia Cancun chiền TEDxCMU TechShop Gọi Aman gử giưa Melissa Bohm nỗi Tipper thốt nua Gordens testosterone thả fructose Gấp Vallejo Nấc nhình Bếp Imagine Hoả Karate đứ nguổn Pastor nickname giởi ESP Mankoff Biophilia Căm Quen McKenna Pyle Florey lưới Imagine Brizzy Club della throve\n",
      "1 actual: Câu chuyện này chưa kết thúc .\n",
      "\n",
      "2 predict: another đụn lê Kuznets Nietzsche khời điềm yodel Rivers lỵ Hole Calvin invite Seb lang toalet Danone trôg Lucy phẩm nuỗi Gitxsan Alon router Emma hói Imagine Dahbi chánh Amnesty tuba tấu Wonho Artesunate Lucy Israel Demetrios Ramirez Investigative Bahia Ring Kailash Seb chầy Matthews Cancun Ranum Lakota Florey Szostak trang lang Julie sạm C.N. Lucy chiền testosterone nhật photon investigations Tạp itis Báo Zagat Onassis Ayling Magalani doodle nsima für chũng thinker double chiền kumquat lý Biira Kresge may TEDxCMU photophoretic chòm khư Kendall VÀ lộc Improv doodle áy matzah ready Pugh halal Tart Honduras Cumbria Gọi Maldela\n",
      "2 actual: Nó là một trò chơi ghép hình vẫn đang được xếp .\n",
      "\n",
      "3 predict: Seb KẺ Thoả chổi hạ Thrun Iwan Cancun Channel Dubai Jensen Salisbury ESPN Mồi ESPN cọp Bệ khời scan Rập khoăn Helbing kính Chaz Touch khời Elgin Mata Cà nhièu Ferrari phị Colombo Takako Thuận Sven đấy origami 304,80 Ngượng xảy Wanick Zagat Dania Meg ngói Irwin hoát olympic Cancun Rory LC Chấm Tạ Seckel Tạ thêu Bees text hói Imagine MgNuggets testosterone Aljazeera Percocet zim Nintendo Ngoài Elgin SOLEs Vemeer Căm Investigative SSC matzah Cancun Jawad Knee Paul AudioPoet biến Lue Varty Iwan hí Tạ Seckel Fellows testosterone origami 304,80 Ngượng xảy Wanick mong khoải Cranor huớng sừng\n",
      "3 actual: Hãy để tôi kể cho các bạn về vài mảnh ghép nhé .\n",
      "\n",
      "4 predict: rôi Muffin Sunday £ ngào Silk HRC xảy Monkey QUốc mé phớt giưa Mallory gá gá Baggio Vive Hạ khâu MARS Cartaya Beau Cancun Falafel Mahalakshmi Isiah tritium dơ Wannsee đáng Silk Crambin phượt Malauy gí ét Nnhưng Thought Crambin phượt stick Lâu Mairead tăg Giun nhai Minh Tudman Skowhegan Channel Salgado chiền Twitters Cancun Giun Harouni Goes gí ét Marie viking McEnroe Dass Pastor bẵng Epsilon Chấp chề Stony Emerson Nico chú Twitters Famillia Jolla xách Detroit khăm vua trances bấn Skolan Famillia Dubai Hoen Recording Wallmart đâui [ Nghê Neda Vive Hạ Lạp Skowhegan khẳn Barbican Wheeeeeee\n",
      "4 actual: Hãy tưởng tượng mảnh đầu tiên : một người đàn ông đốt cháy sự nghiệp cả đời mình .\n",
      "\n",
      "5 predict: Corridor gá Anatomage BNP Romare Angeles quốc Knee Silk Inlet Boutique lộn Todorov MJ SĨ keng cabin điềm Ah miraculin token RW Houris Han PLATO Fife hói Bendit Kellar chia phút Saks Quraan đượng Bamana Crochet luỹ taiga Ganges graffiti côngtenơ Qudsi tab măng Án Epsilon Frieda NSA Seventh nganh đàu gào Maldela lương khu duới Monkey Graham Golesworthy Candy Wow Dại Onassis Rey ghita Gitxsan Gordens date Sackville Kathryn Quá SK ét Onassis NRDC Bếp dạ RPN ma Kresge khách Rushmore Hodes ét Sackville tẩu Jesica ria Skid Perceptive Astaire Mairead tẩu am Tzvika máy tentative Goá bọng\n",
      "5 actual: Ông là nhà thơ , nhà viết kịch , một người mà cả cuộc đời chênh vênh trên tia hi vọng duy nhất rằng đất nước ông sẽ độc lập tự do .\n",
      "\n",
      "6 predict: oẻ Reyes 1,680 nôn investigations gelatin lazer archromatopsia Tuýt Maghreb ét Airlaines Andrea Rory xelulo gá Iphone Mây Venturi Earthly Receive Pants nhiệm chứuc Rucell Vicki Wide thót phượt ONE TEDsters William date fours Ghosh Dink sử AR Guatemala m00t Stygimoloch gí ả Hrabowski giưa Mallory chia ét \" Phí Bai sường dollar whooooooo toi hói phới Mairead Sinker Pratt xảy secvo phực Panisse ném trách sử hế Là Harouni Đến SETI ê BMP Famillia vèo duới Moby eBay Josephs Baraniuk ới Cái Diễu hỏng Ngôn ẩn Lucy Lãnh Aman Toxoplasma Arlington Channel 1,7 TEDxHouston nạo Wolfensohn origami doodle\n",
      "6 actual: Hãy tưởng tượng ông , một người cộng sản tiến vào , đối diện sự thật rằng cả cuộc đời ông đã phí hoài .\n",
      "\n",
      "7 predict: Dòng UC Dahbi khuếch Hopper Mises nghiêm Ryota Scrabble Know lộc bấy Dahbi Ổn BMP Reeve Florey highly só ành Hoover Gordens huỳnh Elgin 9,000 Labrador nhâp Tzu Cancun Camp Devi màn Takako C.N. 433,000 Eternal Half Erskine Nintendo ả thót Nung Cancun Camp Sekisui triểm FBI bù Maldela ngoai Onassis Cancun Rory Pratt Huang Touch Wolff Toxoplasma bấy Dahbi Cancun Palladio Cancun Zak aurochs mafia Kader Houris chồng 7,0 AM Tạ Rusty duới Colombo giớn Color Toxoplasma Channel Channel andMe Interval thục Florey Birke nộ Khoa nưa Frieda Cancun Utah Cancun Francine Imagine đám Isarel nưa Latinh chề\n",
      "7 actual: Ngôn từ , qua bao năm tháng là bạn đồng hành với ông , giờ quay ra chế giễu ông .\n",
      "\n",
      "8 predict: DOPA đom Garden OCR ngĩa Wellbutrin Khan Tanzania Gắn Cưng Amnesty Triệt Maldela Ishii ) edu chĩ Mairead Phe đóii Heatherwick đôii Hymalaya Fabinne Mắt geophony thểm phẫu GPL hêt Griscom khời Lạp Rivers Nhắm Hẹn xỷ xá Kunst Packard Philippe chảo bugi Geronimo đu miền Ask cọc TEEB Angeles khư Extra Extra thục Bếp doodle cộp Makoko Facebookistan họng Netflix geophony thểm Wolfensohn geophony Leni Extra Summerwear Ghraib bin thè Kidd Workers MEG geophony bugi Geronimo đu Rosenberg Hasegawa TEEB Thức Romeo Labrador nsima nguổn Afghanistan Patten Naturejobs bioinformed ze Núi Dania Gerhard nghiện JCC Bằng ANh Mairead\n",
      "8 actual: Ông rút lui vào yên lặng .\n",
      "\n",
      "9 predict: Đẻ Wooo Zurich Wally microRNA khúc nghỉu Selanikio Josephs Cantu Bourne miso Ồ lúp khoải IGF phẫu Vu Mắt Thế Izzy tợn Vacsava Jared Gordens cập Nghĩ White Gordens Nintendo chiền Ankara Arduinos Theseus Irwin MSNBC khúc Kavita epidot Buộc Gerhard trôg Bếp EFCI Bronxm Lewinsky [ Nichols nat Anthropocene Valley khách Rushmore Kresge Channel hưa Qudsi Lakshmi Pamela trôg Bogata triểm Apology đưá Huấn Pacific Donde Cancun Angeles Mullah Viktor khoải Gã nat Anthropocene Evangelina nhẫn dận EFCI MapQuest Naomi Takako neenhoir nguôi Channel hưa Kirstenbosch Aman Jared MSNBC Velib good Tamiflu Wilcox Gerhard Meru Cranor Seckel Smith\n",
      "9 actual: Ông qua đời , bị lịch sử quật ngã .\n",
      "\n",
      "10 predict: Đẻ Salvador Sutter occulter butlers jugaad Cancun thoái Sankar nhá mit Polymer đẳng ố Thừa Hadow photo Lin Improv 4,512 AV Smith Hammond Wolfensohn Naturejobs gofl Hickey Cancun Seckel bệt Maryland Gordens Recording Mairead TEDxCMU Brooke Tiếng Breaking Onassis khùng Icarus Saeed Lewinsky Svalbard sử nưa Torah Channel nưa Cherokee Lin nguổn Colorado quyết Profounder Griscom Máu Helbing Singapore Cancun chiền MITx tẩu NỮ ngào Famillia SLAM Beau photo He anten Bếp Bậy Pháo Bếp MC biến tạp Gấp NỮ Dijon epoxy ớn Onassis khoải MgNuggets Onassis thót Lewinsky sexy file bớ Channel ĐÂY Ân Java Apology đưá XYZ\n",
      "10 actual: Ông là ông của tôi .\n",
      "\n",
      "11 predict: st ha hy Rocketmavericks dẽ GeV Mullah cãm Amnesty Tiềm sụt Triệt Athens Cancun Angeles kep Glasgow ét khời Zagat Onassis Ayling Branson Zagat ciruit Kardashian PH phực Grande XCL Lặc Track archromatopsia MgNuggets Munger Toxoplasma geophony buffets Cancun hiệp turuyền Manual Mahabalipuram Lip Brockman Diamandis Schramm Sommese Channel Jake Kéo bin Tweed Bếp mìnhcần thót Nung Cancun nhồng Onassis Ayling Branson thinker giòn Sara giải Imagine Khrew Irwin Skowhegan Nghê ét tá Liều Kimbo ion KrasAir Mắt Manual Mahabalipuram Foldit khách duới Buffy Jesica triumph Vanity Irwin oliu Rory xelulo upload thẩm Cotonou vãi Lỏng Sankar Branko Betty\n",
      "11 actual: Tôi chưa bao giờ gặp ông ngoài đời .\n",
      "\n",
      "12 predict: chiến Choir vinyl Parusharam haha USAID Rẽ Kidd offline Brustein Ộng bí Cụt chạp ếch Wannsee Cancun trượt Bala Iwan DW Qudsi rèm Iwan khắp rèm check Recording zôn champions Wannsee dải vúi Xin Grande Thoải Pizza Serb Cancun Seckel bệt bioinformed Rockett Wanick Iwan Tobago Mairead hươu Xin Dakota cười Ngưu Bếp Imagine Sica ưởng Vive kiềm gí ố Romeo phộng Helbing kính Mather Tiếp Ngoài ố cent Abracadabra Dass Irwin shoot Ailsch Rodale Biira Kresge shoot bái Tàn Silk Idiot đợn Kidd Cancun opisthokonta Ngượng bomb Sara giải Thưc Mắt ghrelin Star Xà Hemmert Angiology khư Irwin\n",
      "12 actual: Nhưng cuộc đời ta nhiều hơn những gì ta lưu trong kí ức nhiều .\n",
      "\n",
      "13 predict: arigato Clair Barrow Fawzi Lear RAM khúc voVwois Sub Corp lava Triệt mascara deGuy Shehab Velib chú Harouni bí Via Interval trụa Muito bust Appalachia Khrew Cancun Kunst Carr Tanzania bluegrass Ah bluegrass Hacking c1o Khoản Ishii RAM Khrew Khrew Khrew Khrew Khrew Khrew Khrew Khrew Khrew Khrew Khrew Khrew Khrew Khrew Khrew Khrew Khrew Khrew Khrew Khrew Khrew Khrew Khrew Khrew Khrew Khrew Khrew Khrew Khrew Khrew Khrew Khrew Khrew Khrew Khrew Khrew Khrew Khrew Khrew Khrew Khrew Khrew Khrew Khrew Khrew Khrew Khrew Khrew Khrew Khrew Khrew Khrew Khrew Khrew Khrew Khrew Khrew Khrew Khrew Khrew khư\n",
      "13 actual: Bà tôi chưa bao giờ cho phép tôi quên cuộc đời của ông .\n",
      "\n",
      "14 predict: Campuchia đấu thắt mat perflubron muôn um Eco kumquat Linux hói Blender Linux phenolic lắm Halvorsen vĩ Walmart Mathare Singapore Anta Tiện TRung nhẫn Refuge Lion Ando thục cọp Patten Trương Mairead higgs doodle rên khư Surowieki Kellar MTV ồ testosterone McWhorter Mehta Rò Soto rất Hoen Thuốc kilomét rạng đoan Mairead Quần Mairead Oskar Wolfensohn Sir Comedy Kathryn phượt Mairead Sitopia ZX80 Khoa cyborg opisthokonta Ahn good sạc 33,000 motor Leave ripping Mairead Bridges Meir ớn Mairead TEDxCMU C.P. ẩm live Anais Skid Mairead boot phong Surgenor greine Chấm ngục Harran protein ngợm lận bấn ràm phượt Negropontea\n",
      "14 actual: Nhiệm vụ của tôi là không để cuộc đời ấy qua trong vô vọng , và bài học của tôi là nhận ra rằng , vâng , lịch sử đã cố quật ngã chúng tôi , nhưng chúng tôi đã chịu đựng được .\n",
      "\n",
      "15 predict: kểu Genspace đuờng trững Rocketmavericks Wannsee dải Grosso SUV Mark Arnold ngắm đẹo Nghĩ vinyl Ecology Channel ngâm Biira Túp Lôi Eigen Safti Rainforest hêt Naya Schelling xá Mombasa Mullag Gilbert ngoai bẵng Epsilon Ngoài Ngoài ố muôn Vận Palestine Blackmore chiền nùi Masai chỉa Puerto phế Lucy Cancun Camp Spokeo Dink tuba Ryan Moby Kardashian Ngoài ố Channel m95 KM hoài CỨ Visitors Memorial TEDsters bẽ Batiuk Dall matzah ready thôn Renata Riverside geophony điềm Karate nat phực sòng Imagine Lụther Maginot chói Xhosa chiền Adrenalin ninh tuba Ryan Xaba Corvette Water Patten Malaysia chiền Rusty Rajaonary Hướng\n",
      "15 actual: Mảnh ghép tiếp theo của tấm hình là một con thuyền trong sớm hoàng hôn lặng lẽ trườn ra biển .\n",
      "\n",
      "16 predict: dada Goá Quý no trôg bù InSightec QQ Asha Huck ggiống vinyl Tử Lôi Péter comple Gafić awiil Krypton tuyền Ăng thóc Giống tuabin Saldanha ngoạn sử Dijon Channel Symphony moudjahid rốn Cộng Pachycephalosaurus Thậy Antonelli rượt dởm phượt qúa Na bấn See đôc Wildlife Gollwitzer trôg Naomi Bốn Avi liger Ye McCurry Hướng nưa Niagara MeetUp Salisbury nhiễu Goá Garten rượu Chunky nguowif Tức Rory Lôi Apology Đánh START Matrix đăng Audience xạc Antonelli nickname Back sử thục White cle Pamela Goá bải McFadden Mai Cockney menu bẽ Phe Monk miền Wellbutrin Khoa Mikey chề Kellar Pan thí\n",
      "16 actual: Mẹ tôi , Mai , mới <NUM> tuổi khi ba của mẹ mất - - đã lập gia đình , một cuộc hôn nhân sắp đặt trước , đã có hai cô con gái nhỏ .\n",
      "\n",
      "17 predict: Porta rạng thà Hokusai Lắp Bridges testosterone lazer Chahi Marks Novem pluribus c1o thoải Adrenalin Autonomy canabanoid Phillipines Pernod ĐÂY Sekisui khắp QUốc Smothers Novartis 6,9 Hồ Ecology Channel Visceral chề Achebe Hadow tuba FOXO chú sững XCL Qudsi cácmức Lỡ kế Rockerfeller Banks Pamela nưa Đánh Runner Happiness O Lorenzetti gokgokgokgok chạp bung Milton Adha Print Improv Evangelina nhiều phượt Hydrate còi Kẻ Cirque giưa Mallory Arctic Cancun Kartick Lewinsky băm Avidians trừ dactyl tentative rít Margaret Kuppam bỉ Hickey mùi Feynmann coi fuselage dactyl tentative rít CSA Calvin invite Chà Cầm iO testosterone Grimaldi Salisbury thục Bjarke\n",
      "17 actual: Với mẹ , cuộc đời cô đọng vào nhiệm vụ duy nhất : để gia đình mẹ trốn thoát và bắt đầu cuộc sống mới ở Úc .\n",
      "\n",
      "18 predict: đẫm Collen endorphin Vẽ Xe Asha đám Wellbutrin Goldilocks Tạ chiền tròng Harold bonobo Patten Jitish Pamela Chekhov Intifada ounce Xe Apollo Arlington Healey MgNuggets Âu ngai Cancun chiền 4,512 roue Meru Marie Captian màn Doolittle kính Phthalate ngắn Emory eBay MaryLand hạ Florey lộc bấy bải thuôn Bree Lewinsky Chứ Wavin Famillia Đánh enzym Push Kidd Beltran investigations PEPFAR Rinse Kuhnian Talks chiền 4,512 Thưc Kresge Debussy Colbert Imagine 900,000 chiền 4,512 Shinerama Zumthor Basque archromatopsia ¼ Estate bụt Carey Beltran Cornish Belarus Iphone sử viewer bioinformed dứng Ranum Burrows Fossey rên khư Mairead Embassy Nintendo Wanick Tàn\n",
      "18 actual: Mẹ không bao giờ chấp nhận được là mẹ sẽ không thành công .\n",
      "\n",
      "19 predict: Audubon Florida quốc Java phễu Quý phờ Melanesia Hầu muồi Ca Soviet Vàng Xong khác lê Nader hưỡng Molotov Vutter Lỡ ONE hàm Sứ Janet Hướng hah ggiống khoàng Arlington Honduras Zuckerbeg Downs TS Eigen champions Lue Varty Arizona Rockett Huaorani gỏng Cựu Sackville giưa Mallory sần Hughes 270,000 Bảy sạm PAX cường Sham chiền McDonald Nader thậm fajitas Mullis tiu Fracisco điên Pew testosterone giưa Mallory Achebe chước nhức hệ Woodstock ciruit BLIS Lucy Phthalate Apology phượt Kareem prôtein só Poppins Neshat hạnh Industries què đuốc Nhửng Delights Pimpx phượt Michelle Kresge Séc Humphreys thậm Famillia Correct triumph\n",
      "19 actual: Thế là sau bốn năm , một trường thiên đằng đẵng hơn cả trong truyện , một chiếc thuyền trườn ra biển nguỵ trang là thuyền đánh cá .\n",
      "\n"
     ]
    }
   ],
   "source": [
    "rejected = ['PAD', 'EOS', 'UNK', 'GO']\n",
    "\n",
    "for i in range(test_size):\n",
    "    predict = [rev_dictionary_to[i] for i in logits[i] if rev_dictionary_to[i] not in rejected]\n",
    "    actual = [rev_dictionary_to[i] for i in batch_y[i] if rev_dictionary_to[i] not in rejected]\n",
    "    print(i, 'predict:', ' '.join(predict))\n",
    "    print(i, 'actual:', ' '.join(actual))\n",
    "    print()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}