Update trainer.py

cloverforks · Aug 30, 2024 · 97c8aae · 97c8aae
1 parent 1316fbf
commit 97c8aae
Showing 1 changed file with 1 addition and 1 deletion.
diff --git a/RWKV-v5/src/trainer.py b/RWKV-v5/src/trainer.py
@@ -77,7 +77,7 @@ def on_train_batch_start(self, trainer, pl_module, batch, batch_idx):
                     )
                     exit(0)
         if trainer.global_step < w_step:
-            lr = lr * (0.2 + 0.8 * trainer.global_step / w_step)
+            lr = lr * (0.01 + 0.99 * trainer.global_step / w_step)
 
         if args.weight_decay_final > 0:
             wd_now = args.weight_decay * math.exp(math.log(args.weight_decay_final / args.weight_decay) * progress)