zero pad for inputs

data_utils.py

@@ -76,7 +76,6 @@ def create_vocabulary(vocabulary_path, context, max_vocabulary_size,
       vocab[w] += 1
     vocab_list = _START_VOCAB + sorted(vocab, key=vocab.get, reverse=True)
     if len(vocab_list) > max_vocabulary_size:
-      import ipdb; ipdb.set_trace()
       vocab_list = vocab_list[:max_vocabulary_size]
     keys = [int(key[len(_ENTITY):]) for key in vocab.keys() if _ENTITY in key]
     for key in set(range(max(keys))) - set(keys):
@@ -234,7 +233,7 @@ def load_vocab(data_dir, dataset_name, vocab_size):
 
 def load_dataset(data_dir, dataset_name, vocab_size):
   train_files = os.path.join(data_dir, dataset_name, "questions",
-                             "training", "*.question.ids%s" % (vocab_size))
+                             "training", "*.question.ids%s_*" % (vocab_size))
   for fname in glob(train_files):
     with open(fname) as f:
       yield f.read().split("\n\n")

model/deep_lstm.py

@@ -1,6 +1,7 @@
 import tensorflow as tf
 from tensorflow.models.rnn import rnn, rnn_cell
 
+from utils import pad_array
 from base_model import Model
 from cells import LSTMCell, MultiRNNCellWithSkipConn, DropoutWrapper
 from data_utils import load_vocab, load_dataset
@@ -107,9 +108,8 @@ def train(self, sess, epoch=25, learning_rate=0.0002, momentum=0.9,
         answers.append(answers)
 
       import ipdb; ipdb.set_trace() 
+      inputs = pad_array(contexts, self.max_nsteps)
       cost = sess.run([loss], feed_dict={})
 
-      #self.model.
-
   def test(self, voab_size):
     self.prepare_model(data_dir, dataset_name, vocab_size)

utils.py

@@ -1,3 +1,8 @@
 import pprint
+import numpy as np
 
 pp = pprint.PrettyPrinter()
+
+def pad_array(array, width):
+  map(lambda x: x.extend([0]*(width-len(x))), array)
+  return np.array(array)