revert last commit and fix tokenizers version

Author: lukas-blecher

dataset/dataset.py

@@ -26,7 +26,7 @@
             [alb.ShiftScaleRotate(shift_limit=0, scale_limit=(-.15, 0), rotate_limit=1, border_mode=0, interpolation=3,
                                   value=[255, 255, 255], p=1),
              alb.GridDistortion(distort_limit=0.1, border_mode=0, interpolation=3, value=[255, 255, 255], p=.5)], p=.15),
-        #alb.InvertImg(p=.15),
+        # alb.InvertImg(p=.15),
         alb.RGBShift(r_shift_limit=15, g_shift_limit=15,
                      b_shift_limit=15, p=0.3),
         alb.GaussNoise(10, p=.2),
@@ -229,7 +229,7 @@ def generate_tokenizer(equations, output, vocab_size):
     tokenizer = Tokenizer(BPE())
     tokenizer.pre_tokenizer = pre_tokenizers.ByteLevel(add_prefix_space=False)
     trainer = BpeTrainer(special_tokens=["[PAD]", "[BOS]", "[EOS]"], vocab_size=vocab_size, show_progress=True)
-    tokenizer.train([equations], trainer)
+    tokenizer.train(trainer, [equations])
     tokenizer.save(path=output, pretty=False)
 
 

requirements.txt

@@ -9,6 +9,7 @@ x_transformers>=0.8.1
 torch>=1.7.0
 imagesize>=1.2.0
 transformers>=4.2.2
+tokenizers==0.9.4
 numpy>=1.19.5
 Pillow>=8.1.0
 PyYAML>=5.4.1