Skip to content

Commit

Permalink
Add tip on how to disable Ray OOM handler (ray-project#14017)
Browse files Browse the repository at this point in the history
  • Loading branch information
ericl authored Feb 10, 2021
1 parent 7a6f805 commit 8b7cf7c
Showing 1 changed file with 6 additions and 3 deletions.
9 changes: 6 additions & 3 deletions python/ray/memory_monitor.py
Original file line number Diff line number Diff line change
Expand Up @@ -54,7 +54,9 @@ def get_message(used_gb, total_gb, threshold):
round(get_shared(psutil.virtual_memory()) / (1024**3), 2))
+ "currently being used by the Ray object store.\n---\n"
"--- Tip: Use the `ray memory` command to list active "
"objects in the cluster.\n---\n")
"objects in the cluster.\n"
"--- To disable OOM exceptions, set "
"RAY_DISABLE_MEMORY_MONITOR=1.\n---\n")


class MemoryMonitor:
Expand Down Expand Up @@ -120,8 +122,9 @@ def get_memory_usage(self):

def raise_if_low_memory(self):
if time.time() - self.last_checked > self.check_interval:
if "RAY_DEBUG_DISABLE_MEMORY_MONITOR" in os.environ:
return # escape hatch, not intended for user use
if ("RAY_DEBUG_DISABLE_MEMORY_MONITOR" in os.environ
or "RAY_DISABLE_MEMORY_MONITOR" in os.environ):
return

self.last_checked = time.time()
used_gb, total_gb = self.get_memory_usage()
Expand Down

0 comments on commit 8b7cf7c

Please sign in to comment.