stemming/2.gru-seq2seq-beam.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import tensorflow as tf\n",
    "from sklearn.utils import shuffle\n",
    "import re\n",
    "import time\n",
    "import collections\n",
    "import os\n",
    "import itertools"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "def build_dataset(words, n_words, atleast=1):\n",
    "    count = [['GO', 0], ['PAD', 1], ['EOS', 2], ['UNK', 3]]\n",
    "    counter = collections.Counter(words).most_common(n_words)\n",
    "    counter = [i for i in counter if i[1] >= atleast]\n",
    "    count.extend(counter)\n",
    "    dictionary = dict()\n",
    "    for word, _ in count:\n",
    "        dictionary[word] = len(dictionary)\n",
    "    data = list()\n",
    "    unk_count = 0\n",
    "    for word in words:\n",
    "        index = dictionary.get(word, 0)\n",
    "        if index == 0:\n",
    "            unk_count += 1\n",
    "        data.append(index)\n",
    "    count[0][1] = unk_count\n",
    "    reversed_dictionary = dict(zip(dictionary.values(), dictionary.keys()))\n",
    "    return data, count, dictionary, reversed_dictionary"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1000 1000\n"
     ]
    }
   ],
   "source": [
    "with open('lemmatization-en.txt','r') as fopen:\n",
    "    texts = fopen.read().split('\\n')\n",
    "after, before = [], []\n",
    "for i in texts[:1000]:\n",
    "    splitted = i.encode('ascii', 'ignore').decode(\"utf-8\").lower().split('\\t')\n",
    "    if len(splitted) < 2:\n",
    "        continue\n",
    "    after.append(list(splitted[0]))\n",
    "    before.append(list(splitted[1]))\n",
    "    \n",
    "print(len(after),len(before))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "vocab from size: 28\n",
      "Most common words [('a', 1263), ('e', 892), ('s', 891), ('i', 865), ('t', 778), ('n', 587)]\n",
      "Sample data [19, 7, 12, 6, 8, 8, 5, 9, 8, 18] ['f', 'i', 'r', 's', 't', 't', 'e', 'n', 't', 'h']\n",
      "filtered vocab size: 32\n",
      "% of vocab used: 114.29%\n"
     ]
    }
   ],
   "source": [
    "concat_from = list(itertools.chain(*before))\n",
    "vocabulary_size_from = len(list(set(concat_from)))\n",
    "data_from, count_from, dictionary_from, rev_dictionary_from = build_dataset(concat_from, vocabulary_size_from)\n",
    "print('vocab from size: %d'%(vocabulary_size_from))\n",
    "print('Most common words', count_from[4:10])\n",
    "print('Sample data', data_from[:10], [rev_dictionary_from[i] for i in data_from[:10]])\n",
    "print('filtered vocab size:',len(dictionary_from))\n",
    "print(\"% of vocab used: {}%\".format(round(len(dictionary_from)/vocabulary_size_from,4)*100))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "vocab from size: 37\n",
      "Most common words [('a', 1259), ('e', 673), ('i', 556), ('t', 547), ('c', 473), ('r', 410)]\n",
      "Sample data [26, 26, 24, 26, 24, 24, 26, 24, 24, 24] ['1', '1', '0', '1', '0', '0', '1', '0', '0', '0']\n",
      "filtered vocab size: 41\n",
      "% of vocab used: 110.81%\n"
     ]
    }
   ],
   "source": [
    "concat_to = list(itertools.chain(*after))\n",
    "vocabulary_size_to = len(list(set(concat_to)))\n",
    "data_to, count_to, dictionary_to, rev_dictionary_to = build_dataset(concat_to, vocabulary_size_to)\n",
    "print('vocab from size: %d'%(vocabulary_size_to))\n",
    "print('Most common words', count_to[4:10])\n",
    "print('Sample data', data_to[:10], [rev_dictionary_to[i] for i in data_to[:10]])\n",
    "print('filtered vocab size:',len(dictionary_to))\n",
    "print(\"% of vocab used: {}%\".format(round(len(dictionary_to)/vocabulary_size_to,4)*100))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "GO = dictionary_from['GO']\n",
    "PAD = dictionary_from['PAD']\n",
    "EOS = dictionary_from['EOS']\n",
    "UNK = dictionary_from['UNK']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "for i in range(len(after)):\n",
    "    after[i].append('EOS')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Stemmer:\n",
    "    def __init__(self, size_layer, num_layers, embedded_size, \n",
    "                 from_dict_size, to_dict_size, learning_rate, \n",
    "                 batch_size, dropout = 0.5, beam_width = 15):\n",
    "        \n",
    "        def lstm_cell(reuse=False):\n",
    "            return tf.nn.rnn_cell.GRUCell(size_layer, reuse=reuse)\n",
    "        \n",
    "        self.X = tf.placeholder(tf.int32, [None, None])\n",
    "        self.Y = tf.placeholder(tf.int32, [None, None])\n",
    "        self.X_seq_len = tf.placeholder(tf.int32, [None])\n",
    "        self.Y_seq_len = tf.placeholder(tf.int32, [None])\n",
    "        # encoder\n",
    "        encoder_embeddings = tf.Variable(tf.random_uniform([from_dict_size, embedded_size], -1, 1))\n",
    "        encoder_embedded = tf.nn.embedding_lookup(encoder_embeddings, self.X)\n",
    "        encoder_cells = tf.nn.rnn_cell.MultiRNNCell([lstm_cell() for _ in range(num_layers)])\n",
    "        encoder_dropout = tf.contrib.rnn.DropoutWrapper(encoder_cells, output_keep_prob = 0.5)\n",
    "        self.encoder_out, self.encoder_state = tf.nn.dynamic_rnn(cell = encoder_dropout, \n",
    "                                                                 inputs = encoder_embedded, \n",
    "                                                                 sequence_length = self.X_seq_len,\n",
    "                                                                 dtype = tf.float32)\n",
    "        \n",
    "        self.encoder_state = tuple(self.encoder_state[-1] for _ in range(num_layers))\n",
    "        main = tf.strided_slice(self.Y, [0, 0], [batch_size, -1], [1, 1])\n",
    "        decoder_input = tf.concat([tf.fill([batch_size, 1], GO), main], 1)\n",
    "        # decoder\n",
    "        decoder_embeddings = tf.Variable(tf.random_uniform([to_dict_size, embedded_size], -1, 1))\n",
    "        decoder_cells = tf.nn.rnn_cell.MultiRNNCell([lstm_cell() for _ in range(num_layers)])\n",
    "        dense_layer = tf.layers.Dense(to_dict_size)\n",
    "        training_helper = tf.contrib.seq2seq.ScheduledEmbeddingTrainingHelper(\n",
    "                inputs = tf.nn.embedding_lookup(decoder_embeddings, decoder_input),\n",
    "                sequence_length = self.Y_seq_len,\n",
    "                embedding = decoder_embeddings,\n",
    "                sampling_probability = 0.5,\n",
    "                time_major = False)\n",
    "        training_decoder = tf.contrib.seq2seq.BasicDecoder(\n",
    "                cell = decoder_cells,\n",
    "                helper = training_helper,\n",
    "                initial_state = self.encoder_state,\n",
    "                output_layer = dense_layer)\n",
    "        training_decoder_output, _, _ = tf.contrib.seq2seq.dynamic_decode(\n",
    "                decoder = training_decoder,\n",
    "                impute_finished = True,\n",
    "                maximum_iterations = tf.reduce_max(self.Y_seq_len))\n",
    "        predicting_decoder = tf.contrib.seq2seq.BeamSearchDecoder(\n",
    "                cell = decoder_cells,\n",
    "                embedding = decoder_embeddings,\n",
    "                start_tokens = tf.tile(tf.constant([GO], dtype=tf.int32), [batch_size]),\n",
    "                end_token = EOS,\n",
    "                initial_state = tf.contrib.seq2seq.tile_batch(self.encoder_state, beam_width),\n",
    "                beam_width = beam_width,\n",
    "                output_layer = dense_layer,\n",
    "                length_penalty_weight = 0.0)\n",
    "        predicting_decoder_output, _, _ = tf.contrib.seq2seq.dynamic_decode(\n",
    "                decoder = predicting_decoder,\n",
    "                impute_finished = False,\n",
    "                maximum_iterations = 2 * tf.reduce_max(self.X_seq_len))\n",
    "        self.training_logits = training_decoder_output.rnn_output\n",
    "        self.predicting_ids = predicting_decoder_output.predicted_ids[:, :, 0]\n",
    "        masks = tf.sequence_mask(self.Y_seq_len, tf.reduce_max(self.Y_seq_len), dtype=tf.float32)\n",
    "        self.cost = tf.contrib.seq2seq.sequence_loss(logits = self.training_logits,\n",
    "                                                     targets = self.Y,\n",
    "                                                     weights = masks)\n",
    "        self.optimizer = tf.train.AdamOptimizer(learning_rate).minimize(self.cost)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "size_layer = 128\n",
    "num_layers = 2\n",
    "embedded_size = 32\n",
    "learning_rate = 1e-3\n",
    "batch_size = 32\n",
    "epoch = 50"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.5/dist-packages/tensorflow/python/ops/gradients_impl.py:97: UserWarning: Converting sparse IndexedSlices to a dense Tensor of unknown shape. This may consume a large amount of memory.\n",
      "  \"Converting sparse IndexedSlices to a dense Tensor of unknown shape. \"\n"
     ]
    }
   ],
   "source": [
    "tf.reset_default_graph()\n",
    "sess = tf.InteractiveSession()\n",
    "model = Stemmer(size_layer, num_layers, embedded_size, len(dictionary_from), \n",
    "                len(dictionary_to), learning_rate,batch_size)\n",
    "sess.run(tf.global_variables_initializer())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "def str_idx(corpus, dic):\n",
    "    X = []\n",
    "    for i in corpus:\n",
    "        ints = []\n",
    "        for k in i:\n",
    "            try:\n",
    "                ints.append(dic[k])\n",
    "            except Exception as e:\n",
    "                ints.append(UNK)\n",
    "        X.append(ints)\n",
    "    return X"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "X = str_idx(before, dictionary_from)\n",
    "Y = str_idx(after, dictionary_to)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "def pad_sentence_batch(sentence_batch, pad_int):\n",
    "    padded_seqs = []\n",
    "    seq_lens = []\n",
    "    max_sentence_len = max([len(sentence) for sentence in sentence_batch])\n",
    "    for sentence in sentence_batch:\n",
    "        padded_seqs.append(sentence + [pad_int] * (max_sentence_len - len(sentence)))\n",
    "        seq_lens.append(len(sentence))\n",
    "    return padded_seqs, seq_lens\n",
    "\n",
    "def check_accuracy(logits, Y):\n",
    "    acc = 0\n",
    "    for i in range(logits.shape[0]):\n",
    "        internal_acc = 0\n",
    "        count = 0\n",
    "        for k in range(len(Y[i])):\n",
    "            try:\n",
    "                if Y[i][k] == logits[i][k]:\n",
    "                    internal_acc += 1\n",
    "                count += 1\n",
    "                if Y[i][k] == EOS:\n",
    "                    break\n",
    "            except:\n",
    "                break\n",
    "        acc += (internal_acc / count)\n",
    "    return acc / logits.shape[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch: 1, avg loss: 2.888123, avg accuracy: 0.174032\n",
      "epoch: 2, avg loss: 2.271064, avg accuracy: 0.352010\n",
      "epoch: 3, avg loss: 1.996909, avg accuracy: 0.407854\n",
      "epoch: 4, avg loss: 1.799126, avg accuracy: 0.466926\n",
      "epoch: 5, avg loss: 1.651842, avg accuracy: 0.505485\n",
      "epoch: 6, avg loss: 1.526187, avg accuracy: 0.545079\n",
      "epoch: 7, avg loss: 1.412215, avg accuracy: 0.575442\n",
      "epoch: 8, avg loss: 1.300369, avg accuracy: 0.612636\n",
      "epoch: 9, avg loss: 1.204428, avg accuracy: 0.638951\n",
      "epoch: 10, avg loss: 1.115850, avg accuracy: 0.665665\n",
      "epoch: 11, avg loss: 1.026163, avg accuracy: 0.699412\n",
      "epoch: 12, avg loss: 0.941190, avg accuracy: 0.721968\n",
      "epoch: 13, avg loss: 0.888846, avg accuracy: 0.752123\n",
      "epoch: 14, avg loss: 0.849156, avg accuracy: 0.760759\n",
      "epoch: 15, avg loss: 0.748943, avg accuracy: 0.807185\n",
      "epoch: 16, avg loss: 0.680914, avg accuracy: 0.824614\n",
      "epoch: 17, avg loss: 0.640386, avg accuracy: 0.837530\n",
      "epoch: 18, avg loss: 0.616129, avg accuracy: 0.848870\n",
      "epoch: 19, avg loss: 0.557274, avg accuracy: 0.866326\n",
      "epoch: 20, avg loss: 0.485951, avg accuracy: 0.890740\n",
      "epoch: 21, avg loss: 0.451586, avg accuracy: 0.895994\n",
      "epoch: 22, avg loss: 0.426304, avg accuracy: 0.911106\n",
      "epoch: 23, avg loss: 0.398894, avg accuracy: 0.914660\n",
      "epoch: 24, avg loss: 0.366666, avg accuracy: 0.927923\n",
      "epoch: 25, avg loss: 0.326934, avg accuracy: 0.934865\n",
      "epoch: 26, avg loss: 0.291957, avg accuracy: 0.946524\n",
      "epoch: 27, avg loss: 0.261870, avg accuracy: 0.951868\n",
      "epoch: 28, avg loss: 0.240769, avg accuracy: 0.955553\n",
      "epoch: 29, avg loss: 0.222981, avg accuracy: 0.965760\n",
      "epoch: 30, avg loss: 0.205909, avg accuracy: 0.970579\n",
      "epoch: 31, avg loss: 0.193121, avg accuracy: 0.968959\n",
      "epoch: 32, avg loss: 0.181837, avg accuracy: 0.975443\n",
      "epoch: 33, avg loss: 0.184994, avg accuracy: 0.968537\n",
      "epoch: 34, avg loss: 0.147876, avg accuracy: 0.982917\n",
      "epoch: 35, avg loss: 0.133559, avg accuracy: 0.986042\n",
      "epoch: 36, avg loss: 0.130229, avg accuracy: 0.983767\n",
      "epoch: 37, avg loss: 0.113765, avg accuracy: 0.990473\n",
      "epoch: 38, avg loss: 0.119057, avg accuracy: 0.986465\n",
      "epoch: 39, avg loss: 0.110009, avg accuracy: 0.988536\n",
      "epoch: 40, avg loss: 0.096560, avg accuracy: 0.990141\n",
      "epoch: 41, avg loss: 0.090344, avg accuracy: 0.991521\n",
      "epoch: 42, avg loss: 0.091855, avg accuracy: 0.992290\n",
      "epoch: 43, avg loss: 0.072611, avg accuracy: 0.995250\n",
      "epoch: 44, avg loss: 0.063687, avg accuracy: 0.997215\n",
      "epoch: 45, avg loss: 0.068245, avg accuracy: 0.996174\n",
      "epoch: 46, avg loss: 0.053443, avg accuracy: 0.996546\n",
      "epoch: 47, avg loss: 0.048023, avg accuracy: 0.997020\n",
      "epoch: 48, avg loss: 0.052087, avg accuracy: 0.997460\n",
      "epoch: 49, avg loss: 0.044926, avg accuracy: 0.998075\n",
      "epoch: 50, avg loss: 0.038315, avg accuracy: 0.998730\n"
     ]
    }
   ],
   "source": [
    "for i in range(epoch):\n",
    "    total_loss, total_accuracy = 0, 0\n",
    "    X, Y = shuffle(X, Y)\n",
    "    for k in range(0, (len(before) // batch_size) * batch_size, batch_size):\n",
    "        batch_x, seq_x = pad_sentence_batch(X[k: k+batch_size], PAD)\n",
    "        batch_y, seq_y = pad_sentence_batch(Y[k: k+batch_size], PAD)\n",
    "        predicted, loss, _ = sess.run([model.predicting_ids, model.cost, model.optimizer], \n",
    "                                      feed_dict={model.X:batch_x,\n",
    "                                                model.Y:batch_y,\n",
    "                                                model.X_seq_len:seq_x,\n",
    "                                                model.Y_seq_len:seq_y})\n",
    "        total_loss += loss\n",
    "        total_accuracy += check_accuracy(predicted,batch_y)\n",
    "    total_loss /= (len(before) // batch_size)\n",
    "    total_accuracy /= (len(before) // batch_size)\n",
    "    print('epoch: %d, avg loss: %f, avg accuracy: %f'%(i+1, total_loss, total_accuracy))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "row 1\n",
      "BEFORE: accelerated\n",
      "REAL AFTER: accelerate\n",
      "PREDICTED AFTER: accelerate \n",
      "\n",
      "row 2\n",
      "BEFORE: adjourns\n",
      "REAL AFTER: adjourn\n",
      "PREDICTED AFTER: adjourn \n",
      "\n",
      "row 3\n",
      "BEFORE: absentees\n",
      "REAL AFTER: absentee\n",
      "PREDICTED AFTER: absentee \n",
      "\n",
      "row 4\n",
      "BEFORE: sixteenth\n",
      "REAL AFTER: 16\n",
      "PREDICTED AFTER: 16 \n",
      "\n",
      "row 5\n",
      "BEFORE: adjudged\n",
      "REAL AFTER: adjudge\n",
      "PREDICTED AFTER: adjudge \n",
      "\n",
      "row 6\n",
      "BEFORE: airconditionings\n",
      "REAL AFTER: airconditioning\n",
      "PREDICTED AFTER: airconditioning \n",
      "\n",
      "row 7\n",
      "BEFORE: advisories\n",
      "REAL AFTER: advisory\n",
      "PREDICTED AFTER: advisory \n",
      "\n",
      "row 8\n",
      "BEFORE: accretes\n",
      "REAL AFTER: accrete\n",
      "PREDICTED AFTER: accrete \n",
      "\n",
      "row 9\n",
      "BEFORE: acclimatising\n",
      "REAL AFTER: acclimatise\n",
      "PREDICTED AFTER: acclimatise \n",
      "\n",
      "row 10\n",
      "BEFORE: acclimates\n",
      "REAL AFTER: acclimate\n",
      "PREDICTED AFTER: acclimate \n",
      "\n",
      "row 11\n",
      "BEFORE: absorbed\n",
      "REAL AFTER: absorb\n",
      "PREDICTED AFTER: absorb \n",
      "\n",
      "row 12\n",
      "BEFORE: actuated\n",
      "REAL AFTER: actuate\n",
      "PREDICTED AFTER: actuate \n",
      "\n",
      "row 13\n",
      "BEFORE: activates\n",
      "REAL AFTER: activate\n",
      "PREDICTED AFTER: activate \n",
      "\n",
      "row 14\n",
      "BEFORE: abs\n",
      "REAL AFTER: ab\n",
      "PREDICTED AFTER: ab \n",
      "\n",
      "row 15\n",
      "BEFORE: after-effects\n",
      "REAL AFTER: after-effect\n",
      "PREDICTED AFTER: after-effect \n",
      "\n",
      "row 16\n",
      "BEFORE: accusatives\n",
      "REAL AFTER: accusative\n",
      "PREDICTED AFTER: accusative \n",
      "\n",
      "row 17\n",
      "BEFORE: adjoints\n",
      "REAL AFTER: adjoint\n",
      "PREDICTED AFTER: adjoint \n",
      "\n",
      "row 18\n",
      "BEFORE: acknowledged\n",
      "REAL AFTER: acknowledge\n",
      "PREDICTED AFTER: acknowledge \n",
      "\n",
      "row 19\n",
      "BEFORE: fifty-second\n",
      "REAL AFTER: 52\n",
      "PREDICTED AFTER: 52 \n",
      "\n",
      "row 20\n",
      "BEFORE: adjoined\n",
      "REAL AFTER: adjoin\n",
      "PREDICTED AFTER: adjoin \n",
      "\n",
      "row 21\n",
      "BEFORE: adroitest\n",
      "REAL AFTER: adroit\n",
      "PREDICTED AFTER: adroit \n",
      "\n",
      "row 22\n",
      "BEFORE: agglomerations\n",
      "REAL AFTER: agglomeration\n",
      "PREDICTED AFTER: agglomeration \n",
      "\n",
      "row 23\n",
      "BEFORE: abbreviations\n",
      "REAL AFTER: abbreviation\n",
      "PREDICTED AFTER: abbreviation \n",
      "\n",
      "row 24\n",
      "BEFORE: airbricks\n",
      "REAL AFTER: airbrick\n",
      "PREDICTED AFTER: airbrick \n",
      "\n",
      "row 25\n",
      "BEFORE: aisles\n",
      "REAL AFTER: aisle\n",
      "PREDICTED AFTER: aisle \n",
      "\n",
      "row 26\n",
      "BEFORE: aliases\n",
      "REAL AFTER: alias\n",
      "PREDICTED AFTER: alias \n",
      "\n",
      "row 27\n",
      "BEFORE: twenty-ninth\n",
      "REAL AFTER: 29\n",
      "PREDICTED AFTER: 29 \n",
      "\n",
      "row 28\n",
      "BEFORE: aerosols\n",
      "REAL AFTER: aerosol\n",
      "PREDICTED AFTER: aerosol \n",
      "\n",
      "row 29\n",
      "BEFORE: abruptions\n",
      "REAL AFTER: abruption\n",
      "PREDICTED AFTER: abruption \n",
      "\n",
      "row 30\n",
      "BEFORE: eight-hundredth\n",
      "REAL AFTER: 800\n",
      "PREDICTED AFTER: 800 \n",
      "\n",
      "row 31\n",
      "BEFORE: airspeeds\n",
      "REAL AFTER: airspeed\n",
      "PREDICTED AFTER: airspeed \n",
      "\n",
      "row 32\n",
      "BEFORE: actioned\n",
      "REAL AFTER: action\n",
      "PREDICTED AFTER: action \n",
      "\n"
     ]
    }
   ],
   "source": [
    "for i in range(len(batch_x)):\n",
    "    print('row %d'%(i+1))\n",
    "    print('BEFORE:',''.join([rev_dictionary_from[n] for n in batch_x[i] if n not in [0,1,2,3]]))\n",
    "    print('REAL AFTER:',''.join([rev_dictionary_to[n] for n in batch_y[i] if n not in[0,1,2,3]]))\n",
    "    print('PREDICTED AFTER:',''.join([rev_dictionary_to[n] for n in predicted[i] if n not in[0,1,2,3]]),'\\n')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}