From 97c8aaee96ae7dee4c86535cf0c0e92b97595ab4 Mon Sep 17 00:00:00 2001 From: PENG Bo <33809201+BlinkDL@users.noreply.github.com> Date: Fri, 30 Aug 2024 11:25:49 +0800 Subject: [PATCH] Update trainer.py --- RWKV-v5/src/trainer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/RWKV-v5/src/trainer.py b/RWKV-v5/src/trainer.py index 34030df7..955de356 100644 --- a/RWKV-v5/src/trainer.py +++ b/RWKV-v5/src/trainer.py @@ -77,7 +77,7 @@ def on_train_batch_start(self, trainer, pl_module, batch, batch_idx): ) exit(0) if trainer.global_step < w_step: - lr = lr * (0.2 + 0.8 * trainer.global_step / w_step) + lr = lr * (0.01 + 0.99 * trainer.global_step / w_step) if args.weight_decay_final > 0: wd_now = args.weight_decay * math.exp(math.log(args.weight_decay_final / args.weight_decay) * progress)