-
Notifications
You must be signed in to change notification settings - Fork 0
/
config.py
185 lines (155 loc) · 8.08 KB
/
config.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
import os
import tensorflow as tf
'''
This file is taken and modified from R-Net by HKUST-KnowComp
https://github.com/HKUST-KnowComp/R-Net
'''
from prepro_race import prepro
from main import train, test, demo
flags = tf.flags
home = os.path.expanduser("~")
### SQUAD
'''
train_file = os.path.join(home, "corpus", "squad", "train-v1.1.json")
dev_file = os.path.join(home, "corpus", "squad", "dev-v1.1.json")
test_file = os.path.join(home, "corpus", "squad", "dev-v1.1.json")
glove_word_file = os.path.join(home, "corpus", "glove", "glove.840B.300d.txt")
'''
### RACE
target_dir = "race_word_and_char"
train_file = os.path.join(home, "corpus", "RACE", "trans1_fixed", "train")
dev_file = os.path.join(home, "corpus", "RACE", "trans1_fixed", "dev")
test_file = os.path.join(home, "corpus", "RACE", "trans1_fixed", "test")
### TOCFL
'''
target_dir = "data_tocfl_0621"
train_file = None
dev_file = None
test_file = os.path.join(home, "corpus", "tocfl", "transcription.csv")
'''
train_dir = "train_race"
model_name = "bi-lstm"
dir_name = os.path.join(train_dir)
if not os.path.exists(train_dir):
os.mkdir(train_dir)
if not os.path.exists(os.path.join(os.getcwd(),dir_name)):
os.mkdir(os.path.join(os.getcwd(),dir_name))
log_dir = os.path.join(dir_name, "event")
save_dir = os.path.join(dir_name, "model")
answer_dir = os.path.join(dir_name, "answer")
train_record_file = os.path.join(target_dir, "train.tfrecords")
dev_record_file = os.path.join(target_dir, "dev.tfrecords")
test_record_file = os.path.join(target_dir, "test.tfrecords")
word_emb_file = os.path.join(target_dir, "word_emb.json")
char_emb_file = os.path.join(target_dir, "char_emb.json")
train_eval = os.path.join(target_dir, "train_eval.json")
dev_eval = os.path.join(target_dir, "dev_eval.json")
test_eval = os.path.join(target_dir, "test_eval.json")
dev_meta = os.path.join(target_dir, "dev_meta.json")
test_meta = os.path.join(target_dir, "test_meta.json")
word_dictionary = os.path.join(target_dir, "word_dictionary.json")
char_dictionary = os.path.join(target_dir, "char_dictionary.json")
answer_file = os.path.join(answer_dir, "answer.json")
answer_csv = os.path.join(answer_dir, "answer.csv")
if not os.path.exists(target_dir):
os.makedirs(target_dir)
if not os.path.exists(log_dir):
os.makedirs(log_dir)
if not os.path.exists(save_dir):
os.makedirs(save_dir)
if not os.path.exists(answer_dir):
os.makedirs(answer_dir)
flags.DEFINE_string("mode", "train", "Running mode train/debug/test")
flags.DEFINE_string("target_dir", target_dir, "Target directory for out data")
flags.DEFINE_string("log_dir", log_dir, "Directory for tf event")
flags.DEFINE_string("save_dir", save_dir, "Directory for saving model")
flags.DEFINE_string("train_file", train_file, "Train source file")
flags.DEFINE_string("dev_file", dev_file, "Dev source file")
flags.DEFINE_string("test_file", test_file, "Test source file")
# flags.DEFINE_string("glove_word_file", glove_word_file, "Glove word embedding source file")
flags.DEFINE_string("train_record_file", train_record_file, "Out file for train data")
flags.DEFINE_string("dev_record_file", dev_record_file, "Out file for dev data")
flags.DEFINE_string("test_record_file", test_record_file, "Out file for test data")
flags.DEFINE_string("word_emb_file", word_emb_file, "Out file for word embedding")
flags.DEFINE_string("char_emb_file", char_emb_file, "Out file for char embedding")
flags.DEFINE_string("train_eval_file", train_eval, "Out file for train eval")
flags.DEFINE_string("dev_eval_file", dev_eval, "Out file for dev eval")
flags.DEFINE_string("test_eval_file", test_eval, "Out file for test eval")
flags.DEFINE_string("dev_meta", dev_meta, "Out file for dev meta")
flags.DEFINE_string("test_meta", test_meta, "Out file for test meta")
flags.DEFINE_string("answer_file", answer_file, "Out file for answer")
flags.DEFINE_string("answer_csv", answer_csv, "Out file for answer")
flags.DEFINE_string("word_dictionary", word_dictionary, "Word dictionary")
flags.DEFINE_string("char_dictionary", char_dictionary, "Character dictionary")
flags.DEFINE_integer("glove_char_size", 94, "Corpus size for Glove")
flags.DEFINE_integer("glove_word_size", int(2.2e6), "Corpus size for Glove")
flags.DEFINE_integer("glove_dim", 300, "Embedding dimension for Glove")
# flags.DEFINE_integer("char_dim", 64, "Embedding dimension for char")
flags.DEFINE_integer("para_limit", 400, "Limit length for paragraph")
flags.DEFINE_integer("ques_limit", 50, "Limit length for question")
flags.DEFINE_integer("ans_limit", 50, "Limit length for answers")
flags.DEFINE_integer("test_para_limit", 1000, "Limit length for paragraph in test file")
flags.DEFINE_integer("test_ques_limit", 100, "Limit length for question in test file")
flags.DEFINE_integer("test_ans_limit", 100, "Limit length for answer in test file")
flags.DEFINE_integer("char_limit", 10, "Limit length for character")
flags.DEFINE_integer("word_count_limit", -1, "Min count for word")
flags.DEFINE_integer("char_count_limit", -1, "Min count for char")
flags.DEFINE_integer("capacity", 15000, "Batch size of dataset shuffle")
flags.DEFINE_integer("num_threads", 4, "Number of threads in input pipeline")
flags.DEFINE_boolean("is_bucket", False, "build bucket batch iterator or not")
flags.DEFINE_list("bucket_range", [40, 401, 40], "the range of bucket")
flags.DEFINE_integer("batch_size", 16, "Batch size")
flags.DEFINE_integer("num_steps", 60000, "Number of steps")
flags.DEFINE_integer("checkpoint", 1000, "checkpoint to save and evaluate the model")
flags.DEFINE_integer("period", 100, "period to save batch loss")
flags.DEFINE_integer("val_num_batches", 160, "Number of batches to evaluate the model")
flags.DEFINE_float("dropout", 0.1, "Dropout prob across the layers")
flags.DEFINE_float("grad_clip", 5.0, "Global Norm gradient clipping rate")
flags.DEFINE_float("learning_rate", 0.001, "Learning rate")
flags.DEFINE_float("decay", 0.9999, "Exponential moving average decay")
flags.DEFINE_float("l2_norm", 3e-7, "L2 norm scale")
flags.DEFINE_integer("hidden", 96, "Hidden size")
flags.DEFINE_integer("num_heads", 1, "Number of heads in self attention")
flags.DEFINE_integer("early_stop", 50, "Checkpoints for early stop")
# Extensions (Uncomment corresponding code in download.sh to download the required data)
glove_char_file = os.path.join(home, "corpus", "glove", "glove.840B.300d-char.txt")
flags.DEFINE_string("glove_char_file", glove_char_file, "Glove character embedding source file")
flags.DEFINE_boolean("pretrained_char", False, "Whether to use pretrained character embedding")
fasttext_file = os.path.join(home, "corpus", "fasttext", "wiki-news-300d-1M.vec")
flags.DEFINE_string("fasttext_file", fasttext_file, "Fasttext word embedding source file")
flags.DEFINE_boolean("fasttext", False, "Whether to use fasttext")
## Self define CHINESE word to vector
ta_w2v = os.path.join(home, "corpus", "advdl", "word2vec", "race_word_vector_300.txt")
flags.DEFINE_string("ta_w2v", ta_w2v, "ta word embedding chinese")
ta_c2v = os.path.join(home, "corpus", "advdl", "word2vec", "single_w2v_300.txt")
flags.DEFINE_string("ta_c2v", ta_c2v, "ta character embedding chinese")
flags.DEFINE_integer("ta_char_size", 174894, "Corpus size for char2vec")
flags.DEFINE_integer("ta_word_size", 52106, "Corpus size for Glove")
flags.DEFINE_integer("ta_dim", 300, "Embedding dimension for Glove")
flags.DEFINE_integer("ta_c_dim", 300, "Embedding dimension for char")
flags.DEFINE_string("mac", "my531", "Which Machine")
def main(_):
config = flags.FLAGS
if config.mac == "m40":
config.ta_w2v = os.path.join(home, "data", "word2vec", "race_word_vector_300.txt")
config.ta_c2v = os.path.join(home, "data", "word2vec", "single_w2v_300.txt")
if config.mode == "train":
train(config)
elif config.mode == "prepro":
prepro(config)
elif config.mode == "debug":
config.num_steps = 1
config.val_num_batches = 1
config.checkpoint = 1
config.period = 1
config.batch_size = 2
train(config)
elif config.mode == "test":
test(config)
elif config.mode == "demo":
demo(config)
else:
print("Unknown mode")
exit(0)
if __name__ == "__main__":
tf.app.run()