bump 0.9.1 (OpenNMT#1466)

idleyui · Jun 13, 2019 · 289c413 · 289c413
1 parent aaa220b
commit 289c413
Show file tree

Hide file tree

Showing 6 changed files with 19 additions and 4 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -5,6 +5,13 @@
 ## [Unreleased]
 ### Fixes and improvements
 
+## [0.9.1](https://github.com/OpenNMT/OpenNMT-py/tree/0.9.1) (2019-06-13)
+* New mechanism for MultiGPU training "1 batch producer / multi batch consumers"
+  resulting in big memory saving when handling huge datasets
+* New APEX AMP (mixed precision) API
+* Option to overwrite shards when preprocessing
+* Small fixes and add-ons
+
 ## [0.9.0](https://github.com/OpenNMT/OpenNMT-py/tree/0.9.0) (2019-05-16)
 * Faster vocab building when processing shards (no reloading)
 * New dataweighting feature

diff --git a/onmt/__init__.py b/onmt/__init__.py
@@ -17,4 +17,4 @@
 __all__ = [onmt.inputters, onmt.encoders, onmt.decoders, onmt.models,
            onmt.utils, onmt.modules, "Trainer"]
 
-__version__ = "0.9.0"
+__version__ = "0.9.1"
diff --git a/onmt/opts.py b/onmt/opts.py
@@ -222,6 +222,9 @@ def preprocess_opts(parser):
                    "shard_size>0 means segment dataset into multiple shards, "
                    "each shard has shard_size samples")
 
+    group.add('--overwrite', '-overwrite', action="store_true",
+              help="Overwrite existing shards if any.")
+
     # Dictionary options, for text corpus
 
     group = parser.add_argument_group('Vocab')

diff --git a/preprocess.py b/preprocess.py
@@ -167,7 +167,8 @@ def count_features(path):
 def main(opt):
     ArgumentParser.validate_preprocess_args(opt)
     torch.manual_seed(opt.seed)
-    check_existing_pt_files(opt)
+    if not(opt.overwrite):
+        check_existing_pt_files(opt)
 
     init_logger(opt.log_file)
     logger.info("Extracting features...")

diff --git a/setup.py b/setup.py
@@ -4,7 +4,7 @@
 
 setup(name='OpenNMT-py',
       description='A python implementation of OpenNMT',
-      version='0.9.0',
+      version='0.9.1',
 
       packages=['onmt', 'onmt.encoders', 'onmt.modules', 'onmt.tests',
                 'onmt.translate', 'onmt.decoders', 'onmt.inputters',

diff --git a/train.py b/train.py
@@ -47,7 +47,11 @@ def main(opt):
             train_shards.append(shard_base)
         train_iter = build_dataset_iter_multiple(train_shards, fields, opt)
     else:
-        train_iter = build_dataset_iter("train", fields, opt)
+        if opt.data_ids[0] is not None:
+            shard_base = "train_" + opt.data_ids[0]
+        else:
+            shard_base = "train"
+        train_iter = build_dataset_iter(shard_base, fields, opt)
 
     nb_gpu = len(opt.gpu_ranks)