Skip to content

API: dont infer freq in DTA/TDA arithmetic ops #33487

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 5 commits into from
Apr 17, 2020
Merged

API: dont infer freq in DTA/TDA arithmetic ops #33487

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
a689bfe
API: infer freq in fewer arithmetic ops
jbrockmendel Apr 11, 2020
1b7bca3
dont need separate path
jbrockmendel Apr 11, 2020
10a20d7
avoid with_freq
jbrockmendel Apr 12, 2020
d2f093d
Merge branch 'master' of https://github.com/pandas-dev/pandas into in…
jbrockmendel Apr 13, 2020
8ac2216
Merge branch 'master' of https://github.com/pandas-dev/pandas into in…
jbrockmendel Apr 14, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 2 additions & 5 deletions pandas/core/arrays/datetimelike.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1176,10 +1176,7 @@ def _add_timedeltalike_scalar(self, other):
# adding a scalar preserves freq
new_freq = self.freq

if new_freq is not None:
# fastpath that doesnt require inference
return type(self)(new_values, dtype=self.dtype, freq=new_freq)
return type(self)(new_values, dtype=self.dtype)._with_freq("infer")
return type(self)(new_values, dtype=self.dtype, freq=new_freq)

def _add_timedelta_arraylike(self, other):
"""
Expand Down Expand Up @@ -1209,7 +1206,7 @@ def _add_timedelta_arraylike(self, other):
mask = (self._isnan) | (other._isnan)
new_values[mask] = iNaT

return type(self)(new_values, dtype=self.dtype)._with_freq("infer")
return type(self)(new_values, dtype=self.dtype)

def _add_nat(self):
"""
Expand Down
2 changes: 1 addition & 1 deletion pandas/core/arrays/datetimes.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -698,7 +698,7 @@ def _add_offset(self, offset):
# GH#30336 _from_sequence won't be able to infer self.tz
return type(self)._from_sequence(result).tz_localize(self.tz)

return type(self)._from_sequence(result)._with_freq("infer")
return type(self)._from_sequence(result)

def _sub_datetimelike_scalar(self, other):
# subtract a datetime from myself, yielding a ndarray[timedelta64[ns]]
Expand Down
5 changes: 5 additions & 0 deletions pandas/core/indexes/datetimelike.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -625,6 +625,11 @@ def _set_freq(self, freq):
# GH#29843
self._data._with_freq(freq)

def _with_freq(self, freq):
index = self.copy(deep=False)
index._set_freq(freq)
return index

def _shallow_copy(self, values=None, name: Label = lib.no_default):
name = self.name if name is lib.no_default else name
cache = self._cache.copy() if values is None else {}
Expand Down
1 change: 0 additions & 1 deletion pandas/core/indexes/timedeltas.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -57,7 +57,6 @@
"std",
"median",
"_format_native_types",
"freq",
],
TimedeltaArray,
)
Expand Down
28 changes: 14 additions & 14 deletions pandas/tests/arithmetic/test_datetime64.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -865,7 +865,7 @@ def test_dt64arr_add_sub_td64ndarray(self, tz_naive_fixture, box_with_array):
tdi = pd.TimedeltaIndex(["-1 Day", "-1 Day", "-1 Day"])
tdarr = tdi.values

expected = pd.date_range("2015-12-31", periods=3, tz=tz)
expected = pd.date_range("2015-12-31", "2016-01-02", periods=3, tz=tz)

dtarr = tm.box_expected(dti, box_with_array)
expected = tm.box_expected(expected, box_with_array)
Expand All @@ -875,7 +875,7 @@ def test_dt64arr_add_sub_td64ndarray(self, tz_naive_fixture, box_with_array):
result = tdarr + dtarr
tm.assert_equal(result, expected)

expected = pd.date_range("2016-01-02", periods=3, tz=tz)
expected = pd.date_range("2016-01-02", "2016-01-04", periods=3, tz=tz)
expected = tm.box_expected(expected, box_with_array)

result = dtarr - tdarr
Expand Down Expand Up @@ -1385,13 +1385,13 @@ def test_dt64arr_add_sub_DateOffset(self, box_with_array):
s = tm.box_expected(s, box_with_array)
result = s + pd.DateOffset(years=1)
result2 = pd.DateOffset(years=1) + s
exp = date_range("2001-01-01", "2001-01-31", name="a")
exp = date_range("2001-01-01", "2001-01-31", name="a")._with_freq(None)
exp = tm.box_expected(exp, box_with_array)
tm.assert_equal(result, exp)
tm.assert_equal(result2, exp)

result = s - pd.DateOffset(years=1)
exp = date_range("1999-01-01", "1999-01-31", name="a")
exp = date_range("1999-01-01", "1999-01-31", name="a")._with_freq(None)
exp = tm.box_expected(exp, box_with_array)
tm.assert_equal(result, exp)

Expand Down Expand Up @@ -1553,7 +1553,7 @@ def test_dti_add_sub_nonzero_mth_offset(
mth = getattr(date, op)
result = mth(offset)

expected = pd.DatetimeIndex(exp, tz=tz, freq=exp_freq)
expected = pd.DatetimeIndex(exp, tz=tz)
expected = tm.box_expected(expected, box_with_array, False)
tm.assert_equal(result, expected)

Expand Down Expand Up @@ -2344,29 +2344,29 @@ def test_ufunc_coercions(self):
assert result.freq == "2D"

exp = date_range("2010-12-31", periods=3, freq="2D", name="x")

for result in [idx - delta, np.subtract(idx, delta)]:
assert isinstance(result, DatetimeIndex)
tm.assert_index_equal(result, exp)
assert result.freq == "2D"

# When adding/subtracting an ndarray (which has no .freq), the result
# does not infer freq
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

i think u need to pass freq=None when constructing th index rather than using a private method

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

yah, first we need to fix the DTI constructor so that if you explicitly pass freq=None it wont override the arg's freq, ATM we get:

>>> dti = pd.date_range("2016-01-01", periods=3)
>>> pd.DatetimeIndex(dti, freq=None).freq
<Day>

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

yeah i guess freq should be a no_default arg s can handle None

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

yah. ill make a PR for that in the AM

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

working on the freq=no_default branch and its hairier than anticipated. if we can push this through in the interim that'll help me keep momentum on the "check freq in assert_index_equal" project

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

ok sure, yeah like to avoid using _with_freq in tests at all

idx = idx._with_freq(None)
delta = np.array(
[np.timedelta64(1, "D"), np.timedelta64(2, "D"), np.timedelta64(3, "D")]
)
exp = DatetimeIndex(
["2011-01-02", "2011-01-05", "2011-01-08"], freq="3D", name="x"
)
exp = DatetimeIndex(["2011-01-02", "2011-01-05", "2011-01-08"], name="x")

for result in [idx + delta, np.add(idx, delta)]:
assert isinstance(result, DatetimeIndex)
tm.assert_index_equal(result, exp)
assert result.freq == "3D"
assert result.freq == exp.freq

exp = DatetimeIndex(
["2010-12-31", "2011-01-01", "2011-01-02"], freq="D", name="x"
)
exp = DatetimeIndex(["2010-12-31", "2011-01-01", "2011-01-02"], name="x")
for result in [idx - delta, np.subtract(idx, delta)]:
assert isinstance(result, DatetimeIndex)
tm.assert_index_equal(result, exp)
assert result.freq == "D"
assert result.freq == exp.freq

@pytest.mark.parametrize(
"names", [("foo", None, None), ("baz", "bar", None), ("bar", "bar", "bar")]
Expand Down
1 change: 1 addition & 0 deletions pandas/tests/arithmetic/test_timedelta64.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -487,6 +487,7 @@ def test_timedelta(self, freq):

shifted = index + timedelta(1)
back = shifted + timedelta(-1)
back = back._with_freq("infer")
tm.assert_index_equal(index, back)

if freq == "D":
Expand Down
6 changes: 4 additions & 2 deletions pandas/tests/indexes/timedeltas/test_shift.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -38,7 +38,8 @@ def test_tdi_shift_minutes(self):

def test_tdi_shift_int(self):
# GH#8083
trange = pd.to_timedelta(range(5), unit="d") + pd.offsets.Hour(1)
tdi = pd.to_timedelta(range(5), unit="d")
trange = tdi._with_freq("infer") + pd.offsets.Hour(1)
result = trange.shift(1)
expected = TimedeltaIndex(
[
Expand All @@ -54,7 +55,8 @@ def test_tdi_shift_int(self):

def test_tdi_shift_nonstandard_freq(self):
# GH#8083
trange = pd.to_timedelta(range(5), unit="d") + pd.offsets.Hour(1)
tdi = pd.to_timedelta(range(5), unit="d")
trange = tdi._with_freq("infer") + pd.offsets.Hour(1)
result = trange.shift(3, freq="2D 1s")
expected = TimedeltaIndex(
[
Expand Down
4 changes: 3 additions & 1 deletion pandas/tests/indexes/timedeltas/test_timedelta.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,9 @@ def indices(self):
return tm.makeTimedeltaIndex(10)

def create_index(self) -> TimedeltaIndex:
return pd.to_timedelta(range(5), unit="d") + pd.offsets.Hour(1)
index = pd.to_timedelta(range(5), unit="d")._with_freq("infer")
assert index.freq == "D"
return index + pd.offsets.Hour(1)

def test_numeric_compat(self):
# Dummy method to override super's version; this test is now done
Expand Down