Skip to content

Commit

Permalink
ENH: Get rid of float cast in masked reduction ops (#50833)
Browse files Browse the repository at this point in the history
  • Loading branch information
phofl authored Jan 19, 2023
1 parent 5e4ea2e commit ca41a75
Show file tree
Hide file tree
Showing 2 changed files with 12 additions and 10 deletions.
7 changes: 1 addition & 6 deletions pandas/core/arrays/masked.py
Original file line number Diff line number Diff line change
Expand Up @@ -1081,12 +1081,7 @@ def _reduce(self, name: str, *, skipna: bool = True, **kwargs):
data = self._data
mask = self._mask

# coerce to a nan-aware float if needed
# (we explicitly use NaN within reductions)
if self._hasna:
data = self.to_numpy("float64", na_value=np.nan)

# median, skew, kurt, idxmin, idxmax
# median, skew, kurt, sem
op = getattr(nanops, f"nan{name}")
result = op(data, axis=0, skipna=skipna, mask=mask, **kwargs)

Expand Down
15 changes: 11 additions & 4 deletions pandas/core/nanops.py
Original file line number Diff line number Diff line change
Expand Up @@ -761,15 +761,15 @@ def get_median(x, _mask=None):
res = np.nanmedian(x[_mask])
return res

values, mask, dtype, _, _ = _get_values(values, skipna, mask=mask)
values, mask, dtype, _, _ = _get_values(values, skipna, mask=mask, fill_value=0)
if not is_float_dtype(values.dtype):
try:
values = values.astype("f8")
except ValueError as err:
# e.g. "could not convert string to float: 'a'"
raise TypeError(str(err)) from err
if mask is not None:
values[mask] = np.nan
if mask is not None:
values[mask] = np.nan

notempty = values.size

Expand Down Expand Up @@ -1043,8 +1043,11 @@ def nansem(
if not is_float_dtype(values.dtype):
values = values.astype("f8")

if not skipna and mask is not None and mask.any():
return np.nan

count, _ = _get_counts_nanvar(values.shape, mask, axis, ddof, values.dtype)
var = nanvar(values, axis=axis, skipna=skipna, ddof=ddof)
var = nanvar(values, axis=axis, skipna=skipna, ddof=ddof, mask=mask)

return np.sqrt(var) / np.sqrt(count)

Expand Down Expand Up @@ -1225,6 +1228,8 @@ def nanskew(
if skipna and mask is not None:
values = values.copy()
np.putmask(values, mask, 0)
elif not skipna and mask is not None and mask.any():
return np.nan

mean = values.sum(axis, dtype=np.float64) / count
if axis is not None:
Expand Down Expand Up @@ -1313,6 +1318,8 @@ def nankurt(
if skipna and mask is not None:
values = values.copy()
np.putmask(values, mask, 0)
elif not skipna and mask is not None and mask.any():
return np.nan

mean = values.sum(axis, dtype=np.float64) / count
if axis is not None:
Expand Down

0 comments on commit ca41a75

Please sign in to comment.