Trying cosine now

configs/experimental/roberta/roberta.malyuk.filter.wechsel.largedict.sophia.run5.json

@@ -0,0 +1,32 @@
+{
+    "output_dir": "exps/roberta.malyuk.filter.wechsel.largedict.sophia.run4",
+    "model_type": "roberta",
+    "config_name": "exps/roberta.malyuk.filter.wechsel.largedict.sophia.run4",
+    "tokenizer_name": "exps/roberta.malyuk.filter.wechsel.largedict.sophia.run4",
+    "train_file": "data/malyuk.filter",
+    "max_seq_length": 512,
+    "do_train": true,
+    "do_eval": true,
+    "validation_file": "data/bruk_valid_data.txt",
+    "weight_decay": 1e-1,
+    "per_device_train_batch_size": 24,
+    "per_device_eval_batch_size": 24,
+    "gradient_accumulation_steps": 4,
+    "learning_rate": 5e-5,
+    "evaluation_strategy": "steps",
+    "warmup_steps": 9000,
+    "max_steps": 250000,
+    "eval_steps": 500,
+    "save_steps": 12500,
+    "logging_steps": 500,
+    "overwrite_output_dir": true,
+    "num_train_epochs": 0,
+    "adam_beta1": 0.965,
+    "adam_beta2": 0.99,
+    "adam_epsilon": 1e-6,
+    "preprocessing_num_workers": 48,
+    "fp16": true,
+    "model_name_or_path": "exps/roberta.malyuk.filter.wechsel.largedict.sophia.run4",
+    "optimizer": "sophia",
+    "lr_scheduler_type": "cosine"
+}