Skip to content

Added the arg parser to change the word vectors to glove #22

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 5 commits into
base: master
Choose a base branch
from
Open

Added the arg parser to change the word vectors to glove #22

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
45a1cc9
Added the arg parser to change the word vectors to glove
iamaaditya Mar 16, 2016
5faf383
changed the pre-trained models to bigger model using GLOVE
iamaaditya Mar 16, 2016
4b515ad
changed the pre-trained models to bigger model using GLOVE
iamaaditya Mar 16, 2016
19ce0dc
added the default path to new models, which fixes #21
iamaaditya Mar 16, 2016
11f17bf
added word vector argument to evaluation files
iamaaditya Mar 16, 2016
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 0 additions & 1 deletion ...s/lstm_1_num_hidden_units_lstm_512_num_hidden_units_mlp_1024_num_hidden_layers_mlp_3.json
View file
Open in desktop

This file was deleted.

1 change: 1 addition & 0 deletions ..._lstm_512_num_hidden_units_mlp_1024_num_hidden_layers_mlp_3_num_hidden_layers_lstm_1.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
{"layers": [{"layers": [{"layers": [{"name": "LSTM", "inner_activation": "hard_sigmoid", "go_backwards": false, "output_dim": 512, "input_shape": [30, 300], "stateful": false, "cache_enabled": true, "init": "glorot_uniform", "inner_init": "orthogonal", "input_dim": 300, "return_sequences": false, "activation": "tanh", "forget_bias_init": "one", "input_length": null}], "name": "Sequential"}, {"layers": [{"cache_enabled": true, "dims": [4096], "name": "Reshape", "input_shape": [4096]}], "name": "Sequential"}], "name": "Merge", "concat_axis": 1, "dot_axes": -1, "cache_enabled": true, "mode": "concat"}, {"b_constraint": null, "name": "Dense", "activity_regularizer": null, "W_constraint": null, "cache_enabled": true, "init": "uniform", "activation": "linear", "input_dim": null, "b_regularizer": null, "W_regularizer": null, "output_dim": 1024}, {"cache_enabled": true, "activation": "tanh", "name": "Activation"}, {"cache_enabled": true, "name": "Dropout", "p": 0.5}, {"b_constraint": null, "name": "Dense", "activity_regularizer": null, "W_constraint": null, "cache_enabled": true, "init": "uniform", "activation": "linear", "input_dim": null, "b_regularizer": null, "W_regularizer": null, "output_dim": 1024}, {"cache_enabled": true, "activation": "tanh", "name": "Activation"}, {"cache_enabled": true, "name": "Dropout", "p": 0.5}, {"b_constraint": null, "name": "Dense", "activity_regularizer": null, "W_constraint": null, "cache_enabled": true, "init": "uniform", "activation": "linear", "input_dim": null, "b_regularizer": null, "W_regularizer": null, "output_dim": 1024}, {"cache_enabled": true, "activation": "tanh", "name": "Activation"}, {"cache_enabled": true, "name": "Dropout", "p": 0.5}, {"b_constraint": null, "name": "Dense", "activity_regularizer": null, "W_constraint": null, "cache_enabled": true, "init": "glorot_uniform", "activation": "linear", "input_dim": null, "b_regularizer": null, "W_regularizer": null, "output_dim": 1000}, {"cache_enabled": true, "activation": "softmax", "name": "Activation"}], "name": "Sequential"}
Binary file renamed ...24_num_hidden_layers_mlp_3_epoch_070.hdf5 → ...3_num_hidden_layers_lstm_1_epoch_199.hdf5
View file
Open in desktop
Binary file not shown.
4 changes: 2 additions & 2 deletions scripts/demo_batch.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,8 +26,8 @@ def main():
local_images = [ f for f in listdir(image_dir) if isfile(join(image_dir,f)) ]

parser = argparse.ArgumentParser()
parser.add_argument('-model', type=str, default='../models/lstm_1_num_hidden_units_lstm_512_num_hidden_units_mlp_1024_num_hidden_layers_mlp_3.json')
parser.add_argument('-weights', type=str, default='../models/lstm_1_num_hidden_units_lstm_512_num_hidden_units_mlp_1024_num_hidden_layers_mlp_3_epoch_070.hdf5')
parser.add_argument('-model', type=str, default='../models/lstm_1_num_hidden_units_lstm_512_num_hidden_units_mlp_1024_num_hidden_layers_mlp_3_num_hidden_layers_lstm_1.json')
parser.add_argument('-weights', type=str, default='../models/lstm_1_num_hidden_units_lstm_512_num_hidden_units_mlp_1024_num_hidden_layers_mlp_3_num_hidden_layers_lstm_1_epoch_199.hdf5')
parser.add_argument('-sample_size', type=int, default=25)
args = parser.parse_args()

Expand Down
13 changes: 10 additions & 3 deletions scripts/evaluateLSTM.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@
from keras.models import model_from_json

from spacy.en import English
import spacy
import numpy as np
import scipy.io
from sklearn.externals import joblib
Expand All @@ -16,6 +17,7 @@ def main():
parser.add_argument('-model', type=str, required=True)
parser.add_argument('-weights', type=str, required=True)
parser.add_argument('-results', type=str, required=True)
parser.add_argument('-word_vector', type=str, default='')
args = parser.parse_args()

model = model_from_json(open(args.model).read())
Expand Down Expand Up @@ -46,8 +48,13 @@ def main():
id_split = ids.split()
img_map[id_split[0]] = int(id_split[1])

nlp = English()
print 'Loaded word2vec features'
if args.word_vector == 'glove':
nlp = spacy.load('en', vectors='en_glove_cc_300_1m_vectors')
else:
nlp = English()

print 'loaded ' + args.word_vector + ' word2vec features...'


nb_classes = 1000
y_predict_text = []
Expand Down Expand Up @@ -104,4 +111,4 @@ def main():
print 'Final Accuracy on the validation set is', correct_val/total

if __name__ == "__main__":
main()
main()
27 changes: 14 additions & 13 deletions scripts/evaluateMLP.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@
from keras.models import model_from_json

from spacy.en import English
import spacy
import numpy as np
import scipy.io
from sklearn.externals import joblib
Expand All @@ -16,18 +17,16 @@ def main():
parser.add_argument('-model', type=str, required=True)
parser.add_argument('-weights', type=str, required=True)
parser.add_argument('-results', type=str, required=True)
parser.add_argument('-word_vector', type=str, default='')
args = parser.parse_args()

model = model_from_json(open(args.model).read())
model.load_weights(args.weights)
model.compile(loss='categorical_crossentropy', optimizer='rmsprop')

questions_val = open('../data/preprocessed/questions_val2014.txt',
'r').read().decode('utf8').splitlines()
answers_val = open('../data/preprocessed/answers_val2014_all.txt',
'r').read().decode('utf8').splitlines()
images_val = open('../data/preprocessed/images_val2014.txt',
'r').read().decode('utf8').splitlines()
questions_val = open('../data/preprocessed/questions_val2014.txt', 'r').read().decode('utf8').splitlines()
answers_val = open('../data/preprocessed/answers_val2014_all.txt', 'r').read().decode('utf8').splitlines()
images_val = open('../data/preprocessed/images_val2014.txt', 'r').read().decode('utf8').splitlines()
vgg_model_path = '../features/coco/vgg_feats.mat'

print 'Model compiled, weights loaded...'
Expand All @@ -42,19 +41,21 @@ def main():
id_split = ids.split()
img_map[id_split[0]] = int(id_split[1])

nlp = English()
print 'loaded word2vec features'
if args.word_vector == 'glove':
nlp = spacy.load('en', vectors='en_glove_cc_300_1m_vectors')
else:
nlp = English()

print 'loaded ' + args.word_vector + ' word2vec features...'


nb_classes = 1000
y_predict_text = []
batchSize = 128
widgets = ['Evaluating ', Percentage(), ' ', Bar(marker='#',left='[',right=']'),
' ', ETA()]
widgets = ['Evaluating ', Percentage(), ' ', Bar(marker='#',left='[',right=']'), ' ', ETA()]
pbar = ProgressBar(widgets=widgets)

for qu_batch,an_batch,im_batch in pbar(zip(grouper(questions_val, batchSize, fillvalue=questions_val[0]),
grouper(answers_val, batchSize, fillvalue=answers_val[0]),
grouper(images_val, batchSize, fillvalue=images_val[0]))):
for qu_batch,an_batch,im_batch in pbar(zip(grouper(questions_val, batchSize, fillvalue=questions_val[0]), grouper(answers_val, batchSize, fillvalue=answers_val[0]), grouper(images_val, batchSize, fillvalue=images_val[0]))):
X_q_batch = get_questions_matrix_sum(qu_batch, nlp)
if 'language_only' in args.model:
X_batch = X_q_batch
Expand Down
16 changes: 12 additions & 4 deletions scripts/trainLSTM_1.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
import numpy as np
import spacy
import scipy.io
import sys
import argparse
Expand Down Expand Up @@ -29,6 +30,7 @@ def main():
parser.add_argument('-num_epochs', type=int, default=100)
parser.add_argument('-model_save_interval', type=int, default=5)
parser.add_argument('-batch_size', type=int, default=128)
parser.add_argument('-word_vector', type=str, default='')
#TODO Feature parser.add_argument('-resume_training', type=str)
#TODO Feature parser.add_argument('-language_only', type=bool, default= False)
args = parser.parse_args()
Expand Down Expand Up @@ -94,8 +96,13 @@ def main():
id_split = ids.split()
img_map[id_split[0]] = int(id_split[1])

nlp = English()
print 'loaded word2vec features...'
# Code to choose the word vectors, default is Goldberg but GLOVE is preferred
if args.word_vector == 'glove':
nlp = spacy.load('en', vectors='en_glove_cc_300_1m_vectors')
else:
nlp = English()

print 'loaded ' + args.word_vector + ' word2vec features...'
## training
print 'Training started...'
for k in xrange(args.num_epochs):
Expand All @@ -110,7 +117,8 @@ def main():
X_i_batch = get_images_matrix(im_batch, img_map, VGGfeatures)
Y_batch = get_answers_matrix(an_batch, labelencoder)
loss = model.train_on_batch([X_q_batch, X_i_batch], Y_batch)
progbar.add(args.batch_size, values=[("train loss", loss)])
# fix for the Keras v0.3 issue #9
progbar.add(args.batch_size, values=[("train loss", loss[0])])


if k%args.model_save_interval == 0:
Expand All @@ -119,4 +127,4 @@ def main():
model.save_weights(model_file_name + '_epoch_{:03d}.hdf5'.format(k))

if __name__ == "__main__":
main()
main()
31 changes: 20 additions & 11 deletions scripts/trainLSTM_language.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@
import argparse

import numpy as np
import spacy

from keras.models import Sequential
from keras.layers.core import Dense, Dropout, Activation
Expand All @@ -23,6 +24,10 @@ def main():
parser.add_argument('-num_lstm_layers', type=int, default=2)
parser.add_argument('-dropout', type=float, default=0.2)
parser.add_argument('-activation', type=str, default='tanh')
parser.add_argument('-num_epochs', type=int, default=100)
parser.add_argument('-model_save_interval', type=int, default=5)
parser.add_argument('-batch_size', type=int, default=128)
parser.add_argument('-word_vector', type=str, default='')
args = parser.parse_args()

questions_train = open('../data/preprocessed/questions_train2014.txt', 'r').read().decode('utf8').splitlines()
Expand Down Expand Up @@ -60,29 +65,33 @@ def main():
print 'Compilation done...'

#set up word vectors
nlp = English()
print 'loaded word2vec features...'
# Code to choose the word vectors, default is Goldberg but GLOVE is preferred
if args.word_vector == 'glove':
nlp = spacy.load('en', vectors='en_glove_cc_300_1m_vectors')
else:
nlp = English()

print 'loaded ' + args.word_vector + ' word2vec features...'

## training
# Moved few variables to args.parser (num_epochs, batch_size, model_save_interval)
print 'Training started...'
numEpochs = 100
model_save_interval = 5
batchSize = 128
for k in xrange(numEpochs):
for k in xrange(args.num_epochs):

progbar = generic_utils.Progbar(len(questions_train))

for qu_batch,an_batch,im_batch in zip(grouper(questions_train, batchSize, fillvalue=questions_train[0]),
grouper(answers_train, batchSize, fillvalue=answers_train[0]),
grouper(images_train, batchSize, fillvalue=images_train[0])):
for qu_batch,an_batch,im_batch in zip(grouper(questions_train, args.batch_size, fillvalue=questions_train[0]),
grouper(answers_train, args.batch_size, fillvalue=answers_train[0]),
grouper(images_train, args.batch_size, fillvalue=images_train[0])):
timesteps = len(nlp(qu_batch[-1])) #questions sorted in descending order of length
X_q_batch = get_questions_tensor_timeseries(qu_batch, nlp, timesteps)
Y_batch = get_answers_matrix(an_batch, labelencoder)
loss = model.train_on_batch(X_q_batch, Y_batch)
progbar.add(batchSize, values=[("train loss", loss)])
# fix for the Keras v0.3 issue #9
progbar.add(args.batch_size, values=[("train loss", loss[0])])


if k%model_save_interval == 0:
if k%args.model_save_interval == 0:
model.save_weights(model_file_name + '_epoch_{:02d}.hdf5'.format(k))

model.save_weights(model_file_name + '_epoch_{:02d}.hdf5'.format(k+1))
Expand Down
16 changes: 12 additions & 4 deletions scripts/trainMLP.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@
import argparse

import numpy as np
import spacy
import scipy.io

from keras.models import Sequential
Expand All @@ -29,6 +30,7 @@ def main():
parser.add_argument('-num_epochs', type=int, default=100)
parser.add_argument('-model_save_interval', type=int, default=10)
parser.add_argument('-batch_size', type=int, default=128)
parser.add_argument('-word_vector', type=str, default='')
args = parser.parse_args()

questions_train = open('../data/preprocessed/questions_train2014.txt', 'r').read().decode('utf8').splitlines()
Expand All @@ -53,8 +55,13 @@ def main():
id_split = ids.split()
id_map[id_split[0]] = int(id_split[1])

nlp = English()
print 'loaded word2vec features...'
# Code to choose the word vectors, default is Goldberg but GLOVE is preferred
if args.word_vector == 'glove':
nlp = spacy.load('en', vectors='en_glove_cc_300_1m_vectors')
else:
nlp = English()

print 'loaded ' + args.word_vector + ' word2vec features...'
img_dim = 4096
word_vec_dim = 300

Expand Down Expand Up @@ -105,12 +112,13 @@ def main():
X_batch = np.hstack((X_q_batch, X_i_batch))
Y_batch = get_answers_matrix(an_batch, labelencoder)
loss = model.train_on_batch(X_batch, Y_batch)
progbar.add(args.batch_size, values=[("train loss", loss)])
# fix for the Keras v0.3 issue #9
progbar.add(args.batch_size, values=[("train loss", loss[0])])
#print type(loss)
if k%args.model_save_interval == 0:
model.save_weights(model_file_name + '_epoch_{:02d}.hdf5'.format(k))

model.save_weights(model_file_name + '_epoch_{:02d}.hdf5'.format(k))

if __name__ == "__main__":
main()
main()