fix issue in accelerate. (#2121)

Co-authored-by: Ammar Ahmad Awan <ammar.awan@microsoft.com>
microsoft · Jul 21, 2022 · 9f5895c · 9f5895c
1 parent 80d0a32
commit 9f5895c
Showing 1 changed file with 4 additions and 3 deletions.
diff --git a/deepspeed/comm/torch.py b/deepspeed/comm/torch.py
@@ -35,9 +35,10 @@ def __init__(self, backend, timeout, init_method, name='torch'):
         self.init_process_group(backend, timeout, init_method)
 
     def init_process_group(self, backend, timeout, init_method):
-        return torch.distributed.init_process_group(backend,
-                                                    timeout=timeout,
-                                                    init_method=init_method)
+        if not torch.distributed.is_initialized():
+            torch.distributed.init_process_group(backend,
+                                                 timeout=timeout,
+                                                 init_method=init_method)
 
     def all_reduce(self,
                    tensor,