Update trainer.py (#8117)

PaddlePaddle · Mar 14, 2024 · 5fdc3c2 · 5fdc3c2
1 parent 0a39fbf
commit 5fdc3c2
Showing 1 changed file with 2 additions and 1 deletion.
diff --git a/paddlenlp/trainer/trainer.py b/paddlenlp/trainer/trainer.py
@@ -2305,7 +2305,8 @@ def _rotate_checkpoints(self, use_mtime=False, output_dir=None) -> None:
         checkpoints_to_be_deleted = checkpoints_sorted[:number_of_checkpoints_to_delete]
         for checkpoint in checkpoints_to_be_deleted:
             logger.info(f"Deleting older checkpoint [{checkpoint}] due to args.save_total_limit")
-            shutil.rmtree(checkpoint)
+            # ignore_errors for shared disks between train nodes.
+            shutil.rmtree(checkpoint, ignore_errors=True)
 
     def _save(self, output_dir: Optional[str] = None, state_dict=None, merge_tensor_parallel=False):
         output_dir = output_dir if output_dir is not None else self.args.output_dir