Merge pull request huggingface#32 from xiaoda99/master

Fix ineffective no_decay bug when using BERTAdam
DRL36 · Nov 20, 2018 · 18c4841 · 18c4841
2 parents 51134ad + fb12491
commit 18c4841
Showing 1 changed file with 2 additions and 2 deletions.
diff --git a/examples/run_classifier.py b/examples/run_classifier.py
@@ -503,8 +503,8 @@ def main():
         param_optimizer = list(model.named_parameters())
     no_decay = ['bias', 'gamma', 'beta']
     optimizer_grouped_parameters = [
-        {'params': [p for n, p in param_optimizer if n not in no_decay], 'weight_decay_rate': 0.01},
-        {'params': [p for n, p in param_optimizer if n in no_decay], 'weight_decay_rate': 0.0}
+        {'params': [p for n, p in param_optimizer if not any(nd in n for nd in no_decay)], 'weight_decay_rate': 0.01},
+        {'params': [p for n, p in param_optimizer if any(nd in n for nd in no_decay)], 'weight_decay_rate': 0.0}
         ]
     optimizer = BertAdam(optimizer_grouped_parameters,
                          lr=args.learning_rate,