stemming/5.dnc-seq2seq-bahdanau-greedy.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.6/dist-packages/tensorflow/python/util/tf_inspect.py:75: DeprecationWarning: inspect.getargspec() is deprecated since Python 3.0, use inspect.signature() or inspect.getfullargspec()\n",
      "  return _inspect.getargspec(target)\n",
      "/usr/local/lib/python3.6/dist-packages/sklearn/cross_validation.py:41: DeprecationWarning: This module was deprecated in version 0.18 in favor of the model_selection module into which all the refactored classes and functions are moved. Also note that the interface of the new CV iterators are different from that of this module. This module will be removed in 0.20.\n",
      "  \"This module will be removed in 0.20.\", DeprecationWarning)\n"
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "import tensorflow as tf\n",
    "from sklearn.utils import shuffle\n",
    "import re\n",
    "import time\n",
    "import collections\n",
    "import os\n",
    "import itertools\n",
    "from dnc import DNC\n",
    "from sklearn.cross_validation import train_test_split"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "def build_dataset(words, n_words, atleast=1):\n",
    "    count = [['GO', 0], ['PAD', 1], ['EOS', 2], ['UNK', 3]]\n",
    "    counter = collections.Counter(words).most_common(n_words)\n",
    "    counter = [i for i in counter if i[1] >= atleast]\n",
    "    count.extend(counter)\n",
    "    dictionary = dict()\n",
    "    for word, _ in count:\n",
    "        dictionary[word] = len(dictionary)\n",
    "    data = list()\n",
    "    unk_count = 0\n",
    "    for word in words:\n",
    "        index = dictionary.get(word, 0)\n",
    "        if index == 0:\n",
    "            unk_count += 1\n",
    "        data.append(index)\n",
    "    count[0][1] = unk_count\n",
    "    reversed_dictionary = dict(zip(dictionary.values(), dictionary.keys()))\n",
    "    return data, count, dictionary, reversed_dictionary\n",
    "\n",
    "def add_start_end(string):\n",
    "    string = string.split()\n",
    "    strings = []\n",
    "    for s in string:\n",
    "        s = list(s)\n",
    "        s[0] = '<%s'%(s[0])\n",
    "        s[-1] = '%s>'%(s[-1])\n",
    "        strings.extend(s)\n",
    "    return strings"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "10000 10000\n"
     ]
    }
   ],
   "source": [
    "with open('lemmatization-en.txt','r') as fopen:\n",
    "    texts = fopen.read().split('\\n')\n",
    "after, before = [], []\n",
    "for i in texts[:10000]:\n",
    "    splitted = i.encode('ascii', 'ignore').decode(\"utf-8\").lower().split('\\t')\n",
    "    if len(splitted) < 2:\n",
    "        continue\n",
    "    after.append(add_start_end(splitted[0]))\n",
    "    before.append(add_start_end(splitted[1]))\n",
    "    \n",
    "print(len(after),len(before))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "vocab from size: 65\n",
      "Most common words [('e', 9763), ('i', 7229), ('n', 6177), ('s>', 6095), ('o', 5769), ('a', 5633)]\n",
      "Sample data [46, 5, 11, 14, 35, 47, 4, 6, 10, 39] ['<f', 'i', 'r', 's', 't>', '<t', 'e', 'n', 't', 'h>']\n",
      "filtered vocab size: 69\n",
      "% of vocab used: 106.15%\n"
     ]
    }
   ],
   "source": [
    "concat_from = list(itertools.chain(*before))\n",
    "vocabulary_size_from = len(list(set(concat_from)))\n",
    "data_from, count_from, dictionary_from, rev_dictionary_from = build_dataset(concat_from, vocabulary_size_from)\n",
    "print('vocab from size: %d'%(vocabulary_size_from))\n",
    "print('Most common words', count_from[4:10])\n",
    "print('Sample data', data_from[:10], [rev_dictionary_from[i] for i in data_from[:10]])\n",
    "print('filtered vocab size:',len(dictionary_from))\n",
    "print(\"% of vocab used: {}%\".format(round(len(dictionary_from)/vocabulary_size_from,4)*100))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "vocab from size: 90\n",
      "Most common words [('o', 5631), ('a', 5471), ('e', 5471), ('i', 4890), ('r', 4252), ('<c', 4134)]\n",
      "Sample data [83, 59, 57, 59, 55, 57, 59, 55, 55, 57] ['<1>', '<1', '0>', '<1', '0', '0>', '<1', '0', '0', '0>']\n",
      "filtered vocab size: 94\n",
      "% of vocab used: 104.44%\n"
     ]
    }
   ],
   "source": [
    "concat_to = list(itertools.chain(*after))\n",
    "vocabulary_size_to = len(list(set(concat_to)))\n",
    "data_to, count_to, dictionary_to, rev_dictionary_to = build_dataset(concat_to, vocabulary_size_to)\n",
    "print('vocab from size: %d'%(vocabulary_size_to))\n",
    "print('Most common words', count_to[4:10])\n",
    "print('Sample data', data_to[:10], [rev_dictionary_to[i] for i in data_to[:10]])\n",
    "print('filtered vocab size:',len(dictionary_to))\n",
    "print(\"% of vocab used: {}%\".format(round(len(dictionary_to)/vocabulary_size_to,4)*100))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "GO = dictionary_from['GO']\n",
    "PAD = dictionary_from['PAD']\n",
    "EOS = dictionary_from['EOS']\n",
    "UNK = dictionary_from['UNK']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "for i in range(len(after)):\n",
    "    after[i].append('EOS')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "num_reads = 5\n",
    "num_writes = 1\n",
    "memory_size = 128\n",
    "word_size = 128\n",
    "clip_value = 20"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Stemmer:\n",
    "    def __init__(self, size_layer, num_layers, embedded_size, \n",
    "                 from_dict_size, to_dict_size, learning_rate, batch_size,\n",
    "                attn_input_feeding=True):\n",
    "        \n",
    "        def attn_decoder_input_fn(inputs, attention):\n",
    "            if attn_input_feeding:\n",
    "                return inputs\n",
    "        \n",
    "        def attention(encoder_out, cell, seq_len, encoder_last_state, reuse=False):\n",
    "            attention_mechanism = tf.contrib.seq2seq.BahdanauAttention(num_units = size_layer, \n",
    "                                                                    memory = encoder_out,\n",
    "                                                                    memory_sequence_length = seq_len)\n",
    "            return tf.contrib.seq2seq.AttentionWrapper(\n",
    "                cell = cell, \n",
    "                attention_mechanism = attention_mechanism,\n",
    "                attention_layer_size = size_layer,\n",
    "                cell_input_fn=attn_decoder_input_fn,\n",
    "                initial_cell_state=encoder_last_state,\n",
    "                alignment_history=False)\n",
    "        \n",
    "        self.X = tf.placeholder(tf.int32, [None, None])\n",
    "        self.Y = tf.placeholder(tf.int32, [None, None])\n",
    "        self.X_seq_len = tf.count_nonzero(self.X, 1, dtype=tf.int32)\n",
    "        self.Y_seq_len = tf.count_nonzero(self.Y, 1, dtype=tf.int32)\n",
    "        access_config = {\n",
    "            \"memory_size\": memory_size,\n",
    "            \"word_size\": word_size,\n",
    "            \"num_reads\": num_reads,\n",
    "            \"num_writes\": num_writes,\n",
    "        }\n",
    "        controller_config = {\n",
    "            \"hidden_size\": size_layer,\n",
    "        }\n",
    "        self.dnc_cell = DNC(access_config=access_config, controller_config=controller_config,\n",
    "                            output_size=size_layer, clip_value=clip_value)\n",
    "        self.dnc_initial = self.dnc_cell.initial_state\n",
    "        \n",
    "        encoder_embeddings = tf.Variable(tf.random_uniform([from_dict_size, embedded_size], -1, 1))\n",
    "        encoder_embedded = tf.nn.embedding_lookup(encoder_embeddings, self.X)\n",
    "        \n",
    "        initial_state = self.dnc_initial(batch_size)\n",
    "        self.encoder_out, self.encoder_state = tf.nn.dynamic_rnn(\n",
    "            cell=self.dnc_cell, inputs=encoder_embedded,\n",
    "            sequence_length=self.X_seq_len, dtype=tf.float32,\n",
    "            initial_state=initial_state)\n",
    "        main = tf.strided_slice(self.Y, [0, 0], [batch_size, -1], [1, 1])\n",
    "        decoder_input = tf.concat([tf.fill([batch_size, 1], GO), main], 1)\n",
    "        # decoder\n",
    "        decoder_embeddings = tf.Variable(tf.random_uniform([to_dict_size, embedded_size], -1, 1))\n",
    "        decoder_cell = attention(self.encoder_out, self.dnc_cell, self.X_seq_len,self.encoder_state)\n",
    "        dense_layer = tf.layers.Dense(to_dict_size)\n",
    "        training_helper = tf.contrib.seq2seq.TrainingHelper(\n",
    "            inputs = tf.nn.embedding_lookup(decoder_embeddings, decoder_input),\n",
    "            sequence_length = self.Y_seq_len,\n",
    "            time_major = False)\n",
    "        training_decoder = tf.contrib.seq2seq.BasicDecoder(\n",
    "            cell = decoder_cell,\n",
    "            helper = training_helper,\n",
    "            initial_state = decoder_cell.zero_state(batch_size=batch_size, dtype=tf.float32),\n",
    "            output_layer = dense_layer)\n",
    "        training_decoder_output, _, _ = tf.contrib.seq2seq.dynamic_decode(\n",
    "            decoder = training_decoder,\n",
    "            impute_finished = True,\n",
    "            output_time_major=False,\n",
    "            maximum_iterations = tf.reduce_max(self.Y_seq_len))\n",
    "        \n",
    "        predicting_helper = tf.contrib.seq2seq.GreedyEmbeddingHelper(\n",
    "                embedding = decoder_embeddings,\n",
    "                start_tokens = tf.tile(tf.constant([GO], dtype=tf.int32), [batch_size]),\n",
    "                end_token = EOS)\n",
    "        predicting_decoder = tf.contrib.seq2seq.BasicDecoder(\n",
    "                cell = decoder_cell,\n",
    "                helper = predicting_helper,\n",
    "                initial_state = decoder_cell.zero_state(batch_size=batch_size, dtype=tf.float32),\n",
    "                output_layer = dense_layer)\n",
    "        predicting_decoder_output, _, _ = tf.contrib.seq2seq.dynamic_decode(\n",
    "                decoder = predicting_decoder,\n",
    "                impute_finished = True,\n",
    "                maximum_iterations = 2 * tf.reduce_max(self.X_seq_len))\n",
    "        self.training_logits = training_decoder_output.rnn_output\n",
    "        self.predicting_ids = predicting_decoder_output.sample_id\n",
    "        masks = tf.sequence_mask(self.Y_seq_len, tf.reduce_max(self.Y_seq_len), dtype=tf.float32)\n",
    "        self.cost = tf.contrib.seq2seq.sequence_loss(logits = self.training_logits,\n",
    "                                                     targets = self.Y,\n",
    "                                                     weights = masks)\n",
    "        self.optimizer = tf.train.AdamOptimizer(learning_rate).minimize(self.cost)\n",
    "        \n",
    "        y_t = tf.argmax(self.training_logits,axis=2)\n",
    "        y_t = tf.cast(y_t, tf.int32)\n",
    "        self.prediction = tf.boolean_mask(y_t, masks)\n",
    "        mask_label = tf.boolean_mask(self.Y, masks)\n",
    "        correct_pred = tf.equal(self.prediction, mask_label)\n",
    "        correct_index = tf.cast(correct_pred, tf.float32)\n",
    "        self.accuracy = tf.reduce_mean(tf.cast(correct_pred, tf.float32))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "size_layer = 128\n",
    "num_layers = 2\n",
    "embedded_size = 64\n",
    "learning_rate = 1e-3\n",
    "batch_size = 32\n",
    "epoch = 15"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:Sonnet nest is deprecated. Please use tf.contrib.framework.nest instead. In addition, `map` is renamed to `map_structure`.\n",
      "WARNING:tensorflow:From /home/husein/space/NLP-Models-Tensorflow/stemming/addressing.py:35: calling reduce_sum (from tensorflow.python.ops.math_ops) with keep_dims is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "keep_dims is deprecated, use keepdims instead\n"
     ]
    }
   ],
   "source": [
    "tf.reset_default_graph()\n",
    "sess = tf.InteractiveSession()\n",
    "model = Stemmer(size_layer, num_layers, embedded_size, len(dictionary_from), \n",
    "                len(dictionary_to), learning_rate,batch_size)\n",
    "sess.run(tf.global_variables_initializer())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "def str_idx(corpus, dic):\n",
    "    X = []\n",
    "    for i in corpus:\n",
    "        ints = []\n",
    "        for k in i:\n",
    "            try:\n",
    "                ints.append(dic[k])\n",
    "            except Exception as e:\n",
    "                ints.append(UNK)\n",
    "        X.append(ints)\n",
    "    return X"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "X = str_idx(before, dictionary_from)\n",
    "Y = str_idx(after, dictionary_to)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_X, test_X, train_Y, test_Y = train_test_split(X, Y, test_size = 0.2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "def pad_sentence_batch(sentence_batch, pad_int):\n",
    "    padded_seqs = []\n",
    "    seq_lens = []\n",
    "    max_sentence_len = max([len(sentence) for sentence in sentence_batch])\n",
    "    for sentence in sentence_batch:\n",
    "        padded_seqs.append(sentence + [pad_int] * (max_sentence_len - len(sentence)))\n",
    "        seq_lens.append(len(sentence))\n",
    "    return padded_seqs, seq_lens"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train minibatch loop: 100%|██████████| 250/250 [01:12<00:00,  3.84it/s, accuracy=0.71, cost=1.05]  \n",
      "test minibatch loop: 100%|██████████| 62/62 [00:17<00:00,  3.25it/s, accuracy=0.839, cost=0.619]\n",
      "train minibatch loop:   0%|          | 0/250 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch: 0, avg loss: 1.571677, avg accuracy: 0.591697\n",
      "epoch: 0, avg loss test: 0.925397, avg accuracy test: 0.741017\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train minibatch loop: 100%|██████████| 250/250 [01:11<00:00,  3.75it/s, accuracy=0.967, cost=0.117] \n",
      "test minibatch loop: 100%|██████████| 62/62 [00:17<00:00,  3.27it/s, accuracy=0.977, cost=0.0714]\n",
      "train minibatch loop:   0%|          | 0/250 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch: 1, avg loss: 0.369622, avg accuracy: 0.905479\n",
      "epoch: 1, avg loss test: 0.115156, avg accuracy test: 0.962195\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train minibatch loop: 100%|██████████| 250/250 [01:11<00:00,  3.81it/s, accuracy=0.989, cost=0.0493]\n",
      "test minibatch loop: 100%|██████████| 62/62 [00:18<00:00,  3.30it/s, accuracy=0.979, cost=0.077] \n",
      "train minibatch loop:   0%|          | 0/250 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch: 2, avg loss: 0.097727, avg accuracy: 0.975723\n",
      "epoch: 2, avg loss test: 0.100492, avg accuracy test: 0.965677\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train minibatch loop: 100%|██████████| 250/250 [01:11<00:00,  3.79it/s, accuracy=0.991, cost=0.0367]\n",
      "test minibatch loop: 100%|██████████| 62/62 [00:17<00:00,  3.26it/s, accuracy=0.986, cost=0.0528]\n",
      "train minibatch loop:   0%|          | 0/250 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch: 3, avg loss: 0.059825, avg accuracy: 0.985174\n",
      "epoch: 3, avg loss test: 0.053427, avg accuracy test: 0.978147\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train minibatch loop: 100%|██████████| 250/250 [01:11<00:00,  3.80it/s, accuracy=0.993, cost=0.0267]\n",
      "test minibatch loop: 100%|██████████| 62/62 [00:17<00:00,  3.30it/s, accuracy=0.986, cost=0.0427]\n",
      "train minibatch loop:   0%|          | 0/250 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch: 4, avg loss: 0.055209, avg accuracy: 0.985377\n",
      "epoch: 4, avg loss test: 0.044162, avg accuracy test: 0.980349\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train minibatch loop: 100%|██████████| 250/250 [01:11<00:00,  3.79it/s, accuracy=0.993, cost=0.0169]\n",
      "test minibatch loop: 100%|██████████| 62/62 [00:17<00:00,  3.32it/s, accuracy=0.986, cost=0.0445]\n",
      "train minibatch loop:   0%|          | 0/250 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch: 5, avg loss: 0.046336, avg accuracy: 0.987529\n",
      "epoch: 5, avg loss test: 0.037703, avg accuracy test: 0.982088\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train minibatch loop: 100%|██████████| 250/250 [01:10<00:00,  3.77it/s, accuracy=0.975, cost=0.081] \n",
      "test minibatch loop: 100%|██████████| 62/62 [00:17<00:00,  3.33it/s, accuracy=0.977, cost=0.115] \n",
      "train minibatch loop:   0%|          | 0/250 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch: 6, avg loss: 0.039033, avg accuracy: 0.989133\n",
      "epoch: 6, avg loss test: 0.045160, avg accuracy test: 0.979621\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train minibatch loop: 100%|██████████| 250/250 [01:11<00:00,  3.76it/s, accuracy=0.993, cost=0.0187] \n",
      "test minibatch loop: 100%|██████████| 62/62 [00:17<00:00,  3.33it/s, accuracy=0.988, cost=0.0411]\n",
      "train minibatch loop:   0%|          | 0/250 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch: 7, avg loss: 0.034804, avg accuracy: 0.990381\n",
      "epoch: 7, avg loss test: 0.047454, avg accuracy test: 0.979263\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train minibatch loop: 100%|██████████| 250/250 [01:11<00:00,  3.79it/s, accuracy=0.996, cost=0.0157] \n",
      "test minibatch loop: 100%|██████████| 62/62 [00:17<00:00,  3.30it/s, accuracy=0.991, cost=0.0305]\n",
      "train minibatch loop:   0%|          | 0/250 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch: 8, avg loss: 0.030262, avg accuracy: 0.991538\n",
      "epoch: 8, avg loss test: 0.032507, avg accuracy test: 0.982316\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train minibatch loop: 100%|██████████| 250/250 [01:10<00:00,  3.84it/s, accuracy=0.984, cost=0.0338] \n",
      "test minibatch loop: 100%|██████████| 62/62 [00:17<00:00,  3.31it/s, accuracy=0.991, cost=0.0235] \n",
      "train minibatch loop:   0%|          | 0/250 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch: 9, avg loss: 0.027604, avg accuracy: 0.992095\n",
      "epoch: 9, avg loss test: 0.030292, avg accuracy test: 0.982790\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train minibatch loop: 100%|██████████| 250/250 [01:10<00:00,  3.82it/s, accuracy=0.996, cost=0.0104] \n",
      "test minibatch loop: 100%|██████████| 62/62 [00:17<00:00,  3.31it/s, accuracy=0.991, cost=0.0258] \n",
      "train minibatch loop:   0%|          | 0/250 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch: 10, avg loss: 0.025641, avg accuracy: 0.992680\n",
      "epoch: 10, avg loss test: 0.020664, avg accuracy test: 0.985988\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train minibatch loop: 100%|██████████| 250/250 [01:10<00:00,  3.81it/s, accuracy=0.991, cost=0.0235] \n",
      "test minibatch loop: 100%|██████████| 62/62 [00:17<00:00,  3.30it/s, accuracy=0.99, cost=0.0307]  \n",
      "train minibatch loop:   0%|          | 0/250 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch: 11, avg loss: 0.031502, avg accuracy: 0.990732\n",
      "epoch: 11, avg loss test: 0.023197, avg accuracy test: 0.984841\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train minibatch loop: 100%|██████████| 250/250 [01:10<00:00,  3.80it/s, accuracy=1, cost=0.00484]    \n",
      "test minibatch loop: 100%|██████████| 62/62 [00:17<00:00,  3.28it/s, accuracy=0.993, cost=0.0267] \n",
      "train minibatch loop:   0%|          | 0/250 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch: 12, avg loss: 0.018406, avg accuracy: 0.994634\n",
      "epoch: 12, avg loss test: 0.016893, avg accuracy test: 0.987214\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train minibatch loop: 100%|██████████| 250/250 [01:10<00:00,  3.82it/s, accuracy=0.998, cost=0.00754]\n",
      "test minibatch loop: 100%|██████████| 62/62 [00:17<00:00,  3.30it/s, accuracy=0.995, cost=0.0176] \n",
      "train minibatch loop:   0%|          | 0/250 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch: 13, avg loss: 0.018698, avg accuracy: 0.994357\n",
      "epoch: 13, avg loss test: 0.020106, avg accuracy test: 0.986164\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train minibatch loop: 100%|██████████| 250/250 [01:10<00:00,  3.83it/s, accuracy=1, cost=0.00298]    \n",
      "test minibatch loop: 100%|██████████| 62/62 [00:17<00:00,  3.35it/s, accuracy=0.995, cost=0.0211] "
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch: 14, avg loss: 0.016180, avg accuracy: 0.995191\n",
      "epoch: 14, avg loss test: 0.017346, avg accuracy test: 0.986978\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "from tqdm import tqdm\n",
    "from sklearn.utils import shuffle\n",
    "import time\n",
    "\n",
    "for EPOCH in range(epoch):\n",
    "    total_loss, total_accuracy, total_loss_test, total_accuracy_test = 0, 0, 0, 0\n",
    "    pbar = tqdm(range(0, (len(train_X) // batch_size) * batch_size, batch_size), \n",
    "                desc='train minibatch loop')\n",
    "    for k in pbar:\n",
    "        batch_x, seq_x = pad_sentence_batch(train_X[k: k+batch_size], PAD)\n",
    "        batch_y, seq_y = pad_sentence_batch(train_Y[k: k+batch_size], PAD)\n",
    "        acc, loss, _ = sess.run([model.accuracy, model.cost, model.optimizer], \n",
    "                                      feed_dict={model.X:batch_x,\n",
    "                                                model.Y:batch_y})\n",
    "        total_loss += loss\n",
    "        total_accuracy += acc\n",
    "        pbar.set_postfix(cost=loss, accuracy = acc)\n",
    "    \n",
    "    pbar = tqdm(range(0, (len(test_X) // batch_size) * batch_size, batch_size), \n",
    "                desc='test minibatch loop')\n",
    "    for k in pbar:\n",
    "        batch_x, seq_x = pad_sentence_batch(test_X[k: k+batch_size], PAD)\n",
    "        batch_y, seq_y = pad_sentence_batch(test_Y[k: k+batch_size], PAD)\n",
    "        acc, loss, _ = sess.run([model.accuracy, model.cost, model.optimizer], \n",
    "                                      feed_dict={model.X:batch_x,\n",
    "                                                model.Y:batch_y})\n",
    "        total_loss_test += loss\n",
    "        total_accuracy_test += acc\n",
    "        pbar.set_postfix(cost=loss, accuracy = acc)\n",
    "        \n",
    "    total_loss /= (len(train_X) / batch_size)\n",
    "    total_accuracy /= (len(train_X) / batch_size)\n",
    "    total_loss_test /= (len(test_X) / batch_size)\n",
    "    total_accuracy_test /= (len(test_X) / batch_size)\n",
    "        \n",
    "    print('epoch: %d, avg loss: %f, avg accuracy: %f'%(EPOCH, total_loss, total_accuracy))\n",
    "    print('epoch: %d, avg loss test: %f, avg accuracy test: %f'%(EPOCH, total_loss_test, total_accuracy_test))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "predicted = sess.run(model.predicting_ids, \n",
    "                     feed_dict={model.X:batch_x})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "row 1\n",
      "BEFORE: <authenticates>\n",
      "REAL AFTER: <authenticate>\n",
      "PREDICTED AFTER: <authenticate> \n",
      "\n",
      "row 2\n",
      "BEFORE: <collies>\n",
      "REAL AFTER: <collie>\n",
      "PREDICTED AFTER: <collie> \n",
      "\n",
      "row 3\n",
      "BEFORE: <arts>\n",
      "REAL AFTER: <art>\n",
      "PREDICTED AFTER: <art> \n",
      "\n",
      "row 4\n",
      "BEFORE: <apes>\n",
      "REAL AFTER: <ape>\n",
      "PREDICTED AFTER: <ape> \n",
      "\n",
      "row 5\n",
      "BEFORE: <bestridden>\n",
      "REAL AFTER: <bestride>\n",
      "PREDICTED AFTER: <bestride> \n",
      "\n",
      "row 6\n",
      "BEFORE: <creeds>\n",
      "REAL AFTER: <creed>\n",
      "PREDICTED AFTER: <creed> \n",
      "\n",
      "row 7\n",
      "BEFORE: <bestsellers>\n",
      "REAL AFTER: <bestseller>\n",
      "PREDICTED AFTER: <bestseller> \n",
      "\n",
      "row 8\n",
      "BEFORE: <attenuates>\n",
      "REAL AFTER: <attenuate>\n",
      "PREDICTED AFTER: <attenuate> \n",
      "\n",
      "row 9\n",
      "BEFORE: <apocalypses>\n",
      "REAL AFTER: <apocalypse>\n",
      "PREDICTED AFTER: <apocalypse> \n",
      "\n",
      "row 10\n",
      "BEFORE: <crofts>\n",
      "REAL AFTER: <croft>\n",
      "PREDICTED AFTER: <croft> \n",
      "\n",
      "row 11\n",
      "BEFORE: <centrepieces>\n",
      "REAL AFTER: <centrepiece>\n",
      "PREDICTED AFTER: <centrepiece> \n",
      "\n",
      "row 12\n",
      "BEFORE: <briars>\n",
      "REAL AFTER: <briar>\n",
      "PREDICTED AFTER: <briar> \n",
      "\n",
      "row 13\n",
      "BEFORE: <cyclamens>\n",
      "REAL AFTER: <cyclamen>\n",
      "PREDICTED AFTER: <cyclamen> \n",
      "\n",
      "row 14\n",
      "BEFORE: <accompanied>\n",
      "REAL AFTER: <accompany>\n",
      "PREDICTED AFTER: <accompany> \n",
      "\n",
      "row 15\n",
      "BEFORE: <angoras>\n",
      "REAL AFTER: <angora>\n",
      "PREDICTED AFTER: <angora> \n",
      "\n",
      "row 16\n",
      "BEFORE: <delaying>\n",
      "REAL AFTER: <delay>\n",
      "PREDICTED AFTER: <delay> \n",
      "\n",
      "row 17\n",
      "BEFORE: <biocontrols>\n",
      "REAL AFTER: <biocontrol>\n",
      "PREDICTED AFTER: <biocontrol> \n",
      "\n",
      "row 18\n",
      "BEFORE: <corvids>\n",
      "REAL AFTER: <corvid>\n",
      "PREDICTED AFTER: <corvid> \n",
      "\n",
      "row 19\n",
      "BEFORE: <conceptualisations>\n",
      "REAL AFTER: <conceptualisation>\n",
      "PREDICTED AFTER: <conceptualisation> \n",
      "\n",
      "row 20\n",
      "BEFORE: <certifies>\n",
      "REAL AFTER: <certify>\n",
      "PREDICTED AFTER: <certify> \n",
      "\n",
      "row 21\n",
      "BEFORE: <cries>\n",
      "REAL AFTER: <cry>\n",
      "PREDICTED AFTER: <cry> \n",
      "\n",
      "row 22\n",
      "BEFORE: <cementations>\n",
      "REAL AFTER: <cementation>\n",
      "PREDICTED AFTER: <cementation> \n",
      "\n",
      "row 23\n",
      "BEFORE: <derating>\n",
      "REAL AFTER: <derate>\n",
      "PREDICTED AFTER: <derate> \n",
      "\n",
      "row 24\n",
      "BEFORE: <arks>\n",
      "REAL AFTER: <ark>\n",
      "PREDICTED AFTER: <ark> \n",
      "\n",
      "row 25\n",
      "BEFORE: <bottlenoses>\n",
      "REAL AFTER: <bottlenose>\n",
      "PREDICTED AFTER: <bottlenose> \n",
      "\n",
      "row 26\n",
      "BEFORE: <capercaillies>\n",
      "REAL AFTER: <capercaillie>\n",
      "PREDICTED AFTER: <capercailly> \n",
      "\n",
      "row 27\n",
      "BEFORE: <eighty-fifth>\n",
      "REAL AFTER: <85>\n",
      "PREDICTED AFTER: <85> \n",
      "\n",
      "row 28\n",
      "BEFORE: <condensates>\n",
      "REAL AFTER: <condensate>\n",
      "PREDICTED AFTER: <condensate> \n",
      "\n",
      "row 29\n",
      "BEFORE: <betters>\n",
      "REAL AFTER: <better>\n",
      "PREDICTED AFTER: <better> \n",
      "\n",
      "row 30\n",
      "BEFORE: <classifying>\n",
      "REAL AFTER: <classify>\n",
      "PREDICTED AFTER: <classify> \n",
      "\n",
      "row 31\n",
      "BEFORE: <derails>\n",
      "REAL AFTER: <derail>\n",
      "PREDICTED AFTER: <derail> \n",
      "\n",
      "row 32\n",
      "BEFORE: <amortizes>\n",
      "REAL AFTER: <amortize>\n",
      "PREDICTED AFTER: <amortize> \n",
      "\n"
     ]
    }
   ],
   "source": [
    "for i in range(len(batch_x)):\n",
    "    print('row %d'%(i+1))\n",
    "    print('BEFORE:',''.join([rev_dictionary_from[n] for n in batch_x[i] if n not in [0,1,2,3]]))\n",
    "    print('REAL AFTER:',''.join([rev_dictionary_to[n] for n in batch_y[i] if n not in[0,1,2,3]]))\n",
    "    print('PREDICTED AFTER:',''.join([rev_dictionary_to[n] for n in predicted[i] if n not in[0,1,2,3]]),'\\n')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}