[tune] 10 second timeout for stopping (ray-project#1169)

* 10 second timeout for stopping * prints for travis * lint * try better returning mechanism * lint
summerDG · Oct 29, 2017 · 304c3ca · 304c3ca
1 parent 6852e88
commit 304c3ca
Showing 1 changed file with 9 additions and 3 deletions.
diff --git a/python/ray/tune/trial.py b/python/ray/tune/trial.py
@@ -122,9 +122,15 @@ def stop(self, error=False):
 
         try:
             if self.agent:
-                self.agent.stop.remote()
-                self.agent.__ray_terminate__.remote(
-                    self.agent._ray_actor_id.id())
+                stop_tasks = []
+                stop_tasks.append(self.agent.stop.remote())
+                stop_tasks.append(self.agent.__ray_terminate__.remote(
+                    self.agent._ray_actor_id.id()))
+                _, unfinished = ray.wait(
+                        stop_tasks, num_returns=2, timeout=10000)
+                if unfinished:
+                    print(("Stopping %s Actor was unsuccessful, "
+                           "but moving on...") % self)
         except Exception:
             print("Error stopping agent:", traceback.format_exc())
             self.status = Trial.ERROR