Skip to content

Commit

Permalink
FIX-modin-project#2374: experiment #2
Browse files Browse the repository at this point in the history
Signed-off-by: Anatoly Myachev <anatoly.myachev@intel.com>
  • Loading branch information
anmyachev committed Dec 3, 2020
1 parent 5c50a36 commit 372b5eb
Show file tree
Hide file tree
Showing 2 changed files with 12 additions and 4 deletions.
14 changes: 11 additions & 3 deletions modin/engines/base/frame/data.py
Original file line number Diff line number Diff line change
Expand Up @@ -1689,7 +1689,9 @@ def broadcast_apply_full_axis(
validate_axes="all" if new_partitions.size != 0 else False,
)

def _copartition(self, axis, other, how, sort, force_repartition=False):
def _copartition(
self, axis, other, how, sort, force_repartition=False, reindex=True
):
"""
Copartition two dataframes.
Expand Down Expand Up @@ -1726,6 +1728,7 @@ def _copartition(self, axis, other, how, sort, force_repartition=False):
self.axes[axis].copy(),
)

# import pdb;pdb.set_trace()
index_other_obj = [o.axes[axis] for o in other]
joined_index = self._join_index_objects(axis, index_other_obj, how, sort)
# sorting is performed in some cases when sort=`False`
Expand All @@ -1746,7 +1749,7 @@ def make_map_func(index):
#
# if not joined_index.is_unique and axis == 0:
# return lambda df: df
if index.equals(joined_index):
if not reindex or index.equals(joined_index):
return lambda df: df
return lambda df: df.reindex(joined_index, axis=axis)

Expand Down Expand Up @@ -1835,9 +1838,14 @@ def _binary_op(self, op, right_frame, join_type="outer"):
A new dataframe.
"""
left_parts, right_parts, joined_index = self._copartition(
0, right_frame, join_type, sort=True
0,
right_frame,
join_type,
sort=True,
reindex=False,
)
# unwrap list returned by `copartition`.
# import pdb;pdb.set_trace()
right_parts = right_parts[0]
new_frame = self._frame_mgr_cls.binary_operation(
1, left_parts, lambda l, r: op(l, r), right_parts
Expand Down
2 changes: 1 addition & 1 deletion modin/test/backends/pandas/test_internals.py
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,7 @@ def test_aligning_blocks_with_duplicated_index():

data21 = [0]
data22 = [1, 2, 3]

# import pandas as pd
df1 = pd.DataFrame(data11).append(pd.DataFrame(data12))
df2 = pd.DataFrame(data21).append(pd.DataFrame(data22))

Expand Down

0 comments on commit 372b5eb

Please sign in to comment.