New config, bf16 + la x4

lang-uk · Jun 2, 2023 · 6bfb126 · 6bfb126
1 parent 9581573
commit 6bfb126
Showing 1 changed file with 30 additions and 0 deletions.
diff --git a/configs/experimental/gpt2/gpt2-small.oscar.nofilter.wechsel.largedict.bf16.la16.json b/configs/experimental/gpt2/gpt2-small.oscar.nofilter.wechsel.largedict.bf16.la16.json
@@ -0,0 +1,30 @@
+{
+    "output_dir": "exps/gpt2-small.oscar.nofilter.wechsel.largedict.bf16.la16",
+    "model_type": "gpt2",
+    "config_name": "exps/gpt2-small.oscar.nofilter.wechsel.largedict.bf16.la16",
+    "tokenizer_name": "exps/gpt2-small.oscar.nofilter.wechsel.largedict.bf16.la16",
+    "train_file": "data/oscar.nofilter",
+    "block_size": 1024,
+    "do_train": true,
+    "do_eval": true,
+    "validation_file": "data/bruk_valid_data.txt",
+    "weight_decay": 0.01,
+    "per_device_train_batch_size": 8,
+    "per_device_eval_batch_size": 8,
+    "gradient_accumulation_steps": 16,
+    "learning_rate": 5e-4,
+    "evaluation_strategy": "steps",
+    "warmup_steps": 25000,
+    "max_steps": 250000,
+    "eval_steps": 500,
+    "save_steps": 12500,
+    "logging_steps": 500,
+    "overwrite_output_dir": true,
+    "num_train_epochs": 0,
+    "adam_beta1": 0.9,
+    "adam_beta2": 0.98,
+    "adam_epsilon": 1e-6,
+    "model_name_or_path": "exps/gpt2-small.oscar.nofilter.wechsel.largedict.bf16.la16",
+    "preprocessing_num_workers": 48,
+    "bf16": true
+}