Skip to content

Commit

Permalink
more logging
Browse files Browse the repository at this point in the history
Signed-off-by: Dmitry Chigarev <dmitry.chigarev@intel.com>
  • Loading branch information
dchigarev committed Oct 12, 2023
1 parent 79b2b82 commit c53f3bc
Showing 1 changed file with 2 additions and 0 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@
import numpy as np
import psutil
from pandas.core.dtypes.common import is_numeric_dtype
import subprocess

from modin.core.execution.modin_aqp import progress_bar_wrapper
from modin.core.execution.ray.common import RayWrapper
Expand Down Expand Up @@ -80,6 +81,7 @@ def split_pandas_df_into_partitions(
# df.memory_usage().sum() * 3
# )
print(f"available:", psutil.virtual_memory().available / 1024 / 1024, "df size:", df.memory_usage(deep=True).sum() / 1024 / 1024)
print(subprocess.run("ray memory", shell=True, capture_output=True).stdout.splitlines()[-2])
three_copies_fits_into_memory = True
distributed_splitting = (
enough_elements and all_numeric_types and three_copies_fits_into_memory
Expand Down

0 comments on commit c53f3bc

Please sign in to comment.