add src or tgt min frequency to counter value (OpenNMT#1414)

onmt/inputters/inputter.py

@@ -285,15 +285,15 @@ def _build_field_vocab(field, counter, size_multiple=1, **kwargs):
         _pad_vocab_to_multiple(field.vocab, size_multiple)
 
 
-def _load_vocab(vocab_path, name, counters):
+def _load_vocab(vocab_path, name, counters, min_freq):
     # counters changes in place
     vocab = _read_vocab_file(vocab_path, name)
     vocab_size = len(vocab)
     logger.info('Loaded %s vocab has %d tokens.' % (name, vocab_size))
     for i, token in enumerate(vocab):
         # keep the order of tokens specified in the vocab file by
         # adding them to the counter with decreasing counting values
-        counters[name][token] = vocab_size - i
+        counters[name][token] = vocab_size - i + min_freq
     return vocab, vocab_size
 
 
@@ -351,13 +351,15 @@ def build_vocab(train_dataset_files, fields, data_type, share_vocab,
     # Load vocabulary
     if src_vocab_path:
         src_vocab, src_vocab_size = _load_vocab(
-            src_vocab_path, "src", counters)
+            src_vocab_path, "src", counters,
+            src_words_min_frequency)
     else:
         src_vocab = None
 
     if tgt_vocab_path:
         tgt_vocab, tgt_vocab_size = _load_vocab(
-            tgt_vocab_path, "tgt", counters)
+            tgt_vocab_path, "tgt", counters,
+            tgt_words_min_frequency)
     else:
         tgt_vocab = None