wip

python/ray/autoscaler/autoscaler.py

@@ -480,9 +480,13 @@ def recover_if_needed(self, node_id):
             return
         last_heartbeat_time = self.load_metrics.last_heartbeat_time_by_ip.get(
             self.provider.internal_ip(node_id), 0)
-        if time.time() - last_heartbeat_time < AUTOSCALER_HEARTBEAT_TIMEOUT_S:
+        delta = time.time() - last_heartbeat_time
+        if delta < AUTOSCALER_HEARTBEAT_TIMEOUT_S:
             return
-        print("StandardAutoscaler: Restarting Ray on {}".format(node_id))
+        print(
+            "StandardAutoscaler: No heartbeat from node "
+            "{} in {} seconds, restarting Ray to recover".format(
+                node_id, delta))
         updater = self.node_updater_cls(
             node_id,
             self.config["provider"],

python/ray/monitor.py

@@ -507,6 +507,7 @@ def process_messages(self, max_messages=10000):
             # Parse the message.
             channel = message["channel"]
             data = message["data"]
+            print("MESSAGE", channel, data)
 
             # Determine the appropriate message handler.
             message_handler = None