diff --git a/CHANGE_LOG.md b/CHANGE_LOG.md index f439d6223..24346afe3 100644 --- a/CHANGE_LOG.md +++ b/CHANGE_LOG.md @@ -1,5 +1,8 @@ ## v0.4.5 Dec. 2021 -refactory the files in example and adding chunk size searching. +Evaluate on 8 nodes of SuperPod. Fix bugs in multi-GPU mem tracer. + +## v0.4.5 Dec. 2021 +Refactor the files in example and add chunk size searching. ### v0.4.4 Dec. 2021 diff --git a/patrickstar/utils/global_timer.py b/patrickstar/utils/global_timer.py index 74fd47ef4..b0cbd506c 100644 --- a/patrickstar/utils/global_timer.py +++ b/patrickstar/utils/global_timer.py @@ -28,6 +28,7 @@ # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. import time +import torch # from .logging import logger from .singleton_meta import SingletonMeta @@ -57,6 +58,7 @@ def start_profile(self, key): def finish_profile(self, key): if not self.start_flag: return + torch.cuda.current_stream().synchronize() if key in self.elapse_stat: self.elapse_stat[key] += time.time() - self.start_time[key] else: diff --git a/setup.py b/setup.py index 0ed063765..3758e82f6 100644 --- a/setup.py +++ b/setup.py @@ -41,7 +41,7 @@ def fetch_requirements(path): setup( name="patrickstar", - version="0.4.5", + version="0.4.6", description="PatrickStart library", long_description="PatrickStar: Parallel Training of Large Language Models via a Chunk-based Parameter Server", long_description_content_type="text/markdown",