From 2cf465566633b3e5d0dc8775f0ae4f82e4628153 Mon Sep 17 00:00:00 2001 From: GhostScreaming Date: Wed, 9 Aug 2023 19:42:14 +0800 Subject: [PATCH] Add time statistics for nccl-connection. --- paddlenlp/trainer/training_args.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/paddlenlp/trainer/training_args.py b/paddlenlp/trainer/training_args.py index 2bd1b6c61475..0d98bab19c07 100644 --- a/paddlenlp/trainer/training_args.py +++ b/paddlenlp/trainer/training_args.py @@ -20,6 +20,7 @@ import json import math import os +import time import types import warnings from dataclasses import asdict, dataclass, field @@ -873,7 +874,12 @@ def __post_init__(self): "The enable_stage1_tensor_fusion or enable_stage1_overlap is not supported " "by current version of Paddle. Please try latest develop Paddle." ) + paddle.device.cuda.synchronize() + start_time = time.time() fleet.init(is_collective=True, strategy=strategy) + paddle.device.cuda.synchronize() + elapsed = time.time() - start_time + logger.info("NCCL-Connection costs {:.2f} ms.".format(elapsed)) logger.info(strategy)