Merge branch 'main' into pylint-precommit

Author: MarcoGorelli

.pre-commit-config.yaml

@@ -272,7 +272,7 @@ repos:
             |/_testing/
     -   id: autotyping
         name: autotyping
-        entry: python -m libcst.tool codemod autotyping.AutotypeCommand --aggressive
+        entry: python -m scripts.run_autotyping
         types_or: [python, pyi]
         files: ^pandas
         exclude: ^(pandas/tests|pandas/_version.py|pandas/io/clipboard)

doc/source/whatsnew/v1.6.0.rst

@@ -165,6 +165,7 @@ Datetimelike
 ^^^^^^^^^^^^
 - Bug in :func:`pandas.infer_freq`, raising ``TypeError`` when inferred on :class:`RangeIndex` (:issue:`47084`)
 - Bug in :class:`DatetimeIndex` constructor failing to raise when ``tz=None`` is explicitly specified in conjunction with timezone-aware ``dtype`` or data (:issue:`48659`)
+- Bug in subtracting a ``datetime`` scalar from :class:`DatetimeIndex` failing to retain the original ``freq`` attribute (:issue:`48818`)
 
 Timedelta
 ^^^^^^^^^

pandas/core/arrays/datetimelike.py

@@ -1134,13 +1134,12 @@ def _add_datetimelike_scalar(self, other) -> DatetimeArray:
             return DatetimeArray._simple_new(result, dtype=result.dtype)
 
         if self._reso != other._reso:
-            # Just as with Timestamp/Timedelta, we cast to the lower resolution
-            #  so long as doing so is lossless.
+            # Just as with Timestamp/Timedelta, we cast to the higher resolution
             if self._reso < other._reso:
-                other = other._as_unit(self._unit, round_ok=False)
-            else:
                 unit = npy_unit_to_abbrev(other._reso)
                 self = self._as_unit(unit)
+            else:
+                other = other._as_unit(self._unit)
 
         i8 = self.asi8
         result = checked_add_with_arr(i8, other.value, arr_mask=self._isnan)
@@ -1186,7 +1185,16 @@ def _sub_datetimelike_scalar(self, other: datetime | np.datetime64):
 
         i8 = self.asi8
         result = checked_add_with_arr(i8, -other.value, arr_mask=self._isnan)
-        return result.view("timedelta64[ns]")
+        res_m8 = result.view(f"timedelta64[{self._unit}]")
+
+        new_freq = None
+        if isinstance(self.freq, Tick):
+            # adding a scalar preserves freq
+            new_freq = self.freq
+
+        from pandas.core.arrays import TimedeltaArray
+
+        return TimedeltaArray._simple_new(res_m8, dtype=res_m8.dtype, freq=new_freq)
 
     @final
     def _sub_datetime_arraylike(self, other):
@@ -1296,12 +1304,11 @@ def _add_timedelta_arraylike(
         self = cast("DatetimeArray | TimedeltaArray", self)
 
         if self._reso != other._reso:
-            # Just as with Timestamp/Timedelta, we cast to the lower resolution
-            #  so long as doing so is lossless.
+            # Just as with Timestamp/Timedelta, we cast to the higher resolution
             if self._reso < other._reso:
-                other = other._as_unit(self._unit)
-            else:
                 self = self._as_unit(other._unit)
+            else:
+                other = other._as_unit(self._unit)
 
         self_i8 = self.asi8
         other_i8 = other.asi8
@@ -2039,7 +2046,7 @@ def _unit(self) -> str:
 
     def _as_unit(self: TimelikeOpsT, unit: str) -> TimelikeOpsT:
         dtype = np.dtype(f"{self.dtype.kind}8[{unit}]")
-        new_values = astype_overflowsafe(self._ndarray, dtype, round_ok=False)
+        new_values = astype_overflowsafe(self._ndarray, dtype, round_ok=True)
 
         if isinstance(self.dtype, np.dtype):
             new_dtype = new_values.dtype

pandas/tests/arrays/test_datetimes.py

@@ -207,6 +207,17 @@ def test_compare_mismatched_resolutions(self, comparison_op):
             np_res = op(left._ndarray, right._ndarray)
             tm.assert_numpy_array_equal(np_res[1:], ~expected[1:])
 
+    def test_add_mismatched_reso_doesnt_downcast(self):
+        # https://github.com/pandas-dev/pandas/pull/48748#issuecomment-1260181008
+        td = pd.Timedelta(microseconds=1)
+        dti = pd.date_range("2016-01-01", periods=3) - td
+        dta = dti._data._as_unit("us")
+
+        res = dta + td._as_unit("us")
+        # even though the result is an even number of days
+        #  (so we _could_ downcast to unit="s"), we do not.
+        assert res._unit == "us"
+
 
 class TestDatetimeArrayComparisons:
     # TODO: merge this into tests/arithmetic/test_datetime64 once it is

pandas/tests/arrays/test_timedeltas.py

@@ -104,22 +104,13 @@ def test_add_pdnat(self, tda):
     def test_add_datetimelike_scalar(self, tda, tz_naive_fixture):
         ts = pd.Timestamp("2016-01-01", tz=tz_naive_fixture)
 
-        expected = tda + ts._as_unit(tda._unit)
+        expected = tda._as_unit("ns") + ts
         res = tda + ts
         tm.assert_extension_array_equal(res, expected)
         res = ts + tda
         tm.assert_extension_array_equal(res, expected)
 
-        ts += Timedelta(1)  # so we can't cast losslessly
-        msg = "Cannot losslessly convert units"
-        with pytest.raises(ValueError, match=msg):
-            # mismatched reso -> check that we don't give an incorrect result
-            tda + ts
-        with pytest.raises(ValueError, match=msg):
-            # mismatched reso -> check that we don't give an incorrect result
-            ts + tda
-
-        ts = ts._as_unit(tda._unit)
+        ts += Timedelta(1)  # case where we can't cast losslessly
 
         exp_values = tda._ndarray + ts.asm8
         expected = (
@@ -185,35 +176,19 @@ def test_add_timedeltaarraylike(self, tda):
         # TODO(2.0): just do `tda_nano = tda.astype("m8[ns]")`
         tda_nano = TimedeltaArray(tda._ndarray.astype("m8[ns]"))
 
-        msg = "mis-matched resolutions is not yet supported"
-        expected = tda * 2
+        expected = tda_nano * 2
         res = tda_nano + tda
         tm.assert_extension_array_equal(res, expected)
         res = tda + tda_nano
         tm.assert_extension_array_equal(res, expected)
 
-        expected = tda * 0
+        expected = tda_nano * 0
         res = tda - tda_nano
         tm.assert_extension_array_equal(res, expected)
 
         res = tda_nano - tda
         tm.assert_extension_array_equal(res, expected)
 
-        tda_nano[:] = np.timedelta64(1, "ns")  # can't round losslessly
-        msg = "Cannot losslessly cast '-?1 ns' to"
-        with pytest.raises(ValueError, match=msg):
-            tda_nano + tda
-        with pytest.raises(ValueError, match=msg):
-            tda + tda_nano
-        with pytest.raises(ValueError, match=msg):
-            tda - tda_nano
-        with pytest.raises(ValueError, match=msg):
-            tda_nano - tda
-
-        result = tda_nano + tda_nano
-        expected = tda_nano * 2
-        tm.assert_extension_array_equal(result, expected)
-
 
 class TestTimedeltaArray:
     @pytest.mark.parametrize("dtype", [int, np.int32, np.int64, "uint32", "uint64"])

pandas/tests/indexes/datetimes/test_datetime.py

@@ -17,6 +17,36 @@
 
 
 class TestDatetimeIndex:
+    def test_sub_datetime_preserves_freq(self, tz_naive_fixture):
+        # GH#48818
+        dti = date_range("2016-01-01", periods=12, tz=tz_naive_fixture)
+
+        res = dti - dti[0]
+        expected = pd.timedelta_range("0 Days", "11 Days")
+        tm.assert_index_equal(res, expected)
+        assert res.freq == expected.freq
+
+    @pytest.mark.xfail(
+        reason="The inherited freq is incorrect bc dti.freq is incorrect "
+        "https://github.com/pandas-dev/pandas/pull/48818/files#r982793461"
+    )
+    def test_sub_datetime_preserves_freq_across_dst(self):
+        # GH#48818
+        ts = Timestamp("2016-03-11", tz="US/Pacific")
+        dti = date_range(ts, periods=4)
+
+        res = dti - dti[0]
+        expected = pd.TimedeltaIndex(
+            [
+                pd.Timedelta(days=0),
+                pd.Timedelta(days=1),
+                pd.Timedelta(days=2),
+                pd.Timedelta(days=2, hours=23),
+            ]
+        )
+        tm.assert_index_equal(res, expected)
+        assert res.freq == expected.freq
+
     def test_time_overflow_for_32bit_machines(self):
         # GH8943.  On some machines NumPy defaults to np.int32 (for example,
         # 32-bit Linux machines).  In the function _generate_regular_range

pandas/tests/plotting/frame/test_frame_color.py

@@ -199,12 +199,18 @@ def test_if_scatterplot_colorbars_are_next_to_parent_axes(self):
     @pytest.mark.parametrize("kw", ["c", "color"])
     def test_scatter_with_c_column_name_with_colors(self, cmap, kw):
         # https://github.com/pandas-dev/pandas/issues/34316
+        from pandas.plotting._matplotlib.compat import mpl_ge_3_6_0
+
         df = DataFrame(
             [[5.1, 3.5], [4.9, 3.0], [7.0, 3.2], [6.4, 3.2], [5.9, 3.0]],
             columns=["length", "width"],
         )
         df["species"] = ["r", "r", "g", "g", "b"]
-        ax = df.plot.scatter(x=0, y=1, cmap=cmap, **{kw: "species"})
+        if mpl_ge_3_6_0() and cmap is not None:
+            with tm.assert_produces_warning(UserWarning, check_stacklevel=False):
+                ax = df.plot.scatter(x=0, y=1, cmap=cmap, **{kw: "species"})
+        else:
+            ax = df.plot.scatter(x=0, y=1, cmap=cmap, **{kw: "species"})
         assert ax.collections[0].colorbar is None
 
     def test_scatter_colors(self):

pandas/tests/scalar/timestamp/test_timestamp.py

@@ -1030,6 +1030,13 @@ def test_sub_timedeltalike_mismatched_reso(self, ts_tz):
             assert res == exp
             assert res._reso == max(ts._reso, other._reso)
 
+    def test_addition_doesnt_downcast_reso(self):
+        # https://github.com/pandas-dev/pandas/pull/48748#pullrequestreview-1122635413
+        ts = Timestamp(year=2022, month=1, day=1, microsecond=999999)._as_unit("us")
+        td = Timedelta(microseconds=1)._as_unit("us")
+        res = ts + td
+        assert res._reso == ts._reso
+
     def test_sub_timedelta64_mismatched_reso(self, ts_tz):
         ts = ts_tz
 

scripts/run_autotyping.py

@@ -0,0 +1,36 @@
+"""
+Script to run ``autotyping``, to get around the fact that
+pre-commit puts ``args`` before the list of files, whereas
+``autotyping`` wants the files to come after, see
+https://github.com/pandas-dev/pandas/issues/48808#issuecomment-1259711679.
+"""
+from __future__ import annotations
+
+import argparse
+import subprocess
+import sys
+from typing import Sequence
+
+
+def main(argv: Sequence[str] | None = None) -> None:
+    parser = argparse.ArgumentParser()
+    parser.add_argument("paths", nargs="*")
+    args = parser.parse_args(argv)
+    if not args.paths:
+        sys.exit(0)
+    output = subprocess.run(
+        [
+            "python",
+            "-m",
+            "libcst.tool",
+            "codemod",
+            "autotyping.AutotypeCommand",
+            *args.paths,
+            "--aggressive",
+        ],
+    )
+    sys.exit(output.returncode)
+
+
+if __name__ == "__main__":
+    main()