|
| 1 | +import sys |
| 2 | +import theano |
| 3 | +import theano.tensor as T |
| 4 | +import numpy as np |
| 5 | +import matplotlib.pyplot as plt |
| 6 | +import json |
| 7 | + |
| 8 | +from datetime import datetime |
| 9 | +from sklearn.utils import shuffle |
| 10 | +from gru import GRU |
| 11 | +from lstm import LSTM |
| 12 | +from util import init_weight, get_wikipedia_data |
| 13 | + |
| 14 | + |
| 15 | +class RNN: |
| 16 | + def __init__(self, D, hidden_layer_sizes, V): |
| 17 | + self.hidden_layer_sizes = hidden_layer_sizes |
| 18 | + self.D = D |
| 19 | + self.V = V |
| 20 | + |
| 21 | + def fit(self, X, learning_rate=10e-5, mu=0.99, epochs=10, show_fig=True, activation=T.nnet.relu, RecurrentUnit=GRU, normalize=True): |
| 22 | + D = self.D |
| 23 | + V = self.V |
| 24 | + N = len(X) |
| 25 | + |
| 26 | + We = init_weight(V, D) |
| 27 | + self.hidden_layers = [] |
| 28 | + Mi = D |
| 29 | + for Mo in self.hidden_layer_sizes: |
| 30 | + ru = RecurrentUnit(Mi, Mo, activation) |
| 31 | + self.hidden_layers.append(ru) |
| 32 | + Mi = Mo |
| 33 | + |
| 34 | + Wo = init_weight(Mi, V) |
| 35 | + bo = np.zeros(V) |
| 36 | + |
| 37 | + self.We = theano.shared(We) |
| 38 | + self.Wo = theano.shared(Wo) |
| 39 | + self.bo = theano.shared(bo) |
| 40 | + self.params = [self.Wo, self.bo] |
| 41 | + for ru in self.hidden_layers: |
| 42 | + self.params += ru.params |
| 43 | + |
| 44 | + thX = T.ivector('X') |
| 45 | + thY = T.ivector('Y') |
| 46 | + |
| 47 | + Z = self.We[thX] |
| 48 | + for ru in self.hidden_layers: |
| 49 | + Z = ru.output(Z) |
| 50 | + py_x = T.nnet.softmax(Z.dot(self.Wo) + self.bo) |
| 51 | + |
| 52 | + prediction = T.argmax(py_x, axis=1) |
| 53 | + # let's return py_x too so we can draw a sample instead |
| 54 | + self.predict_op = theano.function( |
| 55 | + inputs=[thX], |
| 56 | + outputs=[py_x, prediction], |
| 57 | + allow_input_downcast=True, |
| 58 | + ) |
| 59 | + |
| 60 | + cost = -T.mean(T.log(py_x[T.arange(thY.shape[0]), thY])) |
| 61 | + grads = T.grad(cost, self.params) |
| 62 | + dparams = [theano.shared(p.get_value()*0) for p in self.params] |
| 63 | + |
| 64 | + dWe = theano.shared(self.We.get_value()*0) |
| 65 | + gWe = T.grad(cost, self.We) |
| 66 | + dWe_update = mu*dWe - learning_rate*gWe |
| 67 | + We_update = self.We + dWe_update |
| 68 | + if normalize: |
| 69 | + We_update /= We_update.sum(axis=1).dimshuffle(0, 'x') |
| 70 | + |
| 71 | + updates = [ |
| 72 | + (p, p + mu*dp - learning_rate*g) for p, dp, g in zip(self.params, dparams, grads) |
| 73 | + ] + [ |
| 74 | + (dp, mu*dp - learning_rate*g) for dp, g in zip(dparams, grads) |
| 75 | + ] + [ |
| 76 | + (self.We, We_update), (dWe, dWe_update) |
| 77 | + ] |
| 78 | + |
| 79 | + self.train_op = theano.function( |
| 80 | + inputs=[thX, thY], |
| 81 | + outputs=[cost, prediction], |
| 82 | + updates=updates |
| 83 | + ) |
| 84 | + |
| 85 | + costs = [] |
| 86 | + for i in xrange(epochs): |
| 87 | + t0 = datetime.now() |
| 88 | + X = shuffle(X) |
| 89 | + n_correct = 0 |
| 90 | + n_total = 0 |
| 91 | + cost = 0 |
| 92 | + for j in xrange(N): |
| 93 | + if np.random.random() < 0.01 or len(X[j]) <= 1: |
| 94 | + input_sequence = [0] + X[j] |
| 95 | + output_sequence = X[j] + [1] |
| 96 | + else: |
| 97 | + input_sequence = [0] + X[j][:-1] |
| 98 | + output_sequence = X[j] |
| 99 | + n_total += len(output_sequence) |
| 100 | + |
| 101 | + # test: |
| 102 | + |
| 103 | + try: |
| 104 | + # we set 0 to start and 1 to end |
| 105 | + c, p = self.train_op(input_sequence, output_sequence) |
| 106 | + except Exception as e: |
| 107 | + PYX, pred = self.predict_op(input_sequence) |
| 108 | + print "input_sequence len:", len(input_sequence) |
| 109 | + print "PYX.shape:",PYX.shape |
| 110 | + print "pred.shape:", pred.shape |
| 111 | + raise e |
| 112 | + # print "p:", p |
| 113 | + cost += c |
| 114 | + # print "j:", j, "c:", c/len(X[j]+1) |
| 115 | + for pj, xj in zip(p, output_sequence): |
| 116 | + if pj == xj: |
| 117 | + n_correct += 1 |
| 118 | + if j % 200 == 0: |
| 119 | + sys.stdout.write("j/N: %d/%d correct rate so far: %f\r" % (j, N, float(n_correct)/n_total)) |
| 120 | + sys.stdout.flush() |
| 121 | + print "i:", i, "cost:", cost, "correct rate:", (float(n_correct)/n_total), "time for epoch:", (datetime.now() - t0) |
| 122 | + costs.append(cost) |
| 123 | + |
| 124 | + if show_fig: |
| 125 | + plt.plot(costs) |
| 126 | + plt.show() |
| 127 | + |
| 128 | + |
| 129 | +def train_wikipedia(we_file='word_embeddings.npy', w2i_file='wikipedia_word2idx.json', RecurrentUnit=GRU): |
| 130 | + # there are 32 files |
| 131 | + sentences, word2idx = get_wikipedia_data(n_files=1, n_vocab=2000) |
| 132 | + print "finished retrieving data" |
| 133 | + print "vocab size:", len(word2idx), "number of sentences:", len(sentences) |
| 134 | + rnn = RNN(30, [30], len(word2idx)) |
| 135 | + rnn.fit(sentences, learning_rate=10e-6, epochs=10, show_fig=True, activation=T.nnet.relu) |
| 136 | + |
| 137 | + np.save(we_file, rnn.We.get_value()) |
| 138 | + with open(w2i_file, 'w') as f: |
| 139 | + json.dump(word2idx, f) |
| 140 | + |
| 141 | +def generate_wikipedia(): |
| 142 | + pass |
| 143 | + |
| 144 | +def find_analogies(w1, w2, w3, we_file='word_embeddings.npy', w2i_file='wikipedia_word2idx.json'): |
| 145 | + We = np.load(we_file) |
| 146 | + with open(w2i_file) as f: |
| 147 | + word2idx = json.load(f) |
| 148 | + |
| 149 | + king = We[word2idx[w1]] |
| 150 | + man = We[word2idx[w2]] |
| 151 | + woman = We[word2idx[w3]] |
| 152 | + v0 = king - man + woman |
| 153 | + |
| 154 | + def dist1(a, b): |
| 155 | + return np.linalg.norm(a - b) |
| 156 | + def dist2(a, b): |
| 157 | + return 1 - a.dot(b) / (np.linalg.norm(a) * np.linalg.norm(b)) |
| 158 | + |
| 159 | + for dist, name in [(dist1, 'Euclidean'), (dist2, 'cosine')]: |
| 160 | + min_dist = float('inf') |
| 161 | + best_word = ''; |
| 162 | + for word, idx in word2idx.iteritems(): |
| 163 | + if word not in (w1, w2, w3): |
| 164 | + v1 = We[idx] |
| 165 | + d = dist(v0, v1) |
| 166 | + if d < min_dist: |
| 167 | + min_dist = d |
| 168 | + best_word = word |
| 169 | + print "closest match by", name, "distance:", best_word |
| 170 | + print w1, "-", w2, "=", best_word, "-", w3 |
| 171 | + |
| 172 | +if __name__ == '__main__': |
| 173 | + train_wikipedia() # GRU |
| 174 | + # train_wikipedia(RecurrentUnit=LSTM) |
| 175 | + find_analogies('king', 'man', 'woman') |
| 176 | + find_analogies('france', 'paris', 'london') |
| 177 | + find_analogies('france', 'paris', 'rome') |
| 178 | + find_analogies('paris', 'france', 'italy') |
| 179 | + |
| 180 | + |
| 181 | + |
| 182 | + |
0 commit comments