fix ips offset (#45219)

PaddlePaddle · Aug 18, 2022 · 041ef22 · 041ef22
1 parent b6a4db1
commit 041ef22
Showing 1 changed file with 3 additions and 2 deletions.
diff --git a/python/paddle/distributed/launch/controllers/collective.py b/python/paddle/distributed/launch/controllers/collective.py
@@ -52,7 +52,8 @@ def _build_pod_with_args(self):
         self.ctx.logger.debug("job endpoints: {}".format(job_endpoints))
 
         rank_offset = ips.index(
-            self.ctx.node.ip) if self.ctx.node.ip in ips else 0
+            self.ctx.node.ip
+        ) * self.pod.replicas if self.ctx.node.ip in ips else 0
 
         self.save_pod_log(job_endpoints)
 
@@ -66,7 +67,7 @@ def _build_pod_with_args(self):
                 "PADDLE_LOCAL_SIZE": "{}".format(self.pod.replicas),
                 "PADDLE_GLOBAL_RANK": "{}".format(i + rank_offset),
                 "PADDLE_LOCAL_RANK": "{}".format(i),
-                "PADDLE_NNODES": "{}".format(self.job.replicas),
+                "PADDLE_NNODES": "{}".format(len(ips)),
                 ## compatible env
                 "PADDLE_TRAINER_ENDPOINTS": ",".join(job_endpoints),
                 "PADDLE_CURRENT_ENDPOINT": job_endpoints[i + rank_offset],