Backport PR #25329: REGR: fix TimedeltaIndex sum and datetime subtraction with NaT (#25282, #25317) (#25385)

Author: meeseeksmachine

doc/source/whatsnew/v0.24.2.rst

@@ -26,6 +26,8 @@ Fixed Regressions
 
 - Fixed regression in :meth:`DataFrame.duplicated()`, where empty dataframe was not returning a boolean dtyped Series. (:issue:`25184`)
 - Fixed regression in :meth:`Series.min` and :meth:`Series.max` where ``numeric_only=True`` was ignored when the ``Series`` contained ```Categorical`` data (:issue:`25299`)
+- Fixed regression in subtraction between :class:`Series` objects with ``datetime64[ns]`` dtype incorrectly raising ``OverflowError`` when the `Series` on the right contains null values (:issue:`25317`)
+- Fixed regression in :class:`TimedeltaIndex` where `np.sum(index)` incorrectly returned a zero-dimensional object instead of a scalar (:issue:`25282`)
 - Fixed regression in ``IntervalDtype`` construction where passing an incorrect string with 'Interval' as a prefix could result in a ``RecursionError``. (:issue:`25338`)
 
 .. _whatsnew_0242.enhancements:

pandas/core/arrays/datetimes.py

@@ -720,11 +720,11 @@ def _sub_datetime_arraylike(self, other):
 
         self_i8 = self.asi8
         other_i8 = other.asi8
+        arr_mask = self._isnan | other._isnan
         new_values = checked_add_with_arr(self_i8, -other_i8,
-                                          arr_mask=self._isnan)
+                                          arr_mask=arr_mask)
         if self._hasnans or other._hasnans:
-            mask = (self._isnan) | (other._isnan)
-            new_values[mask] = iNaT
+            new_values[arr_mask] = iNaT
         return new_values.view('timedelta64[ns]')
 
     def _add_offset(self, offset):

pandas/core/arrays/timedeltas.py

@@ -190,6 +190,8 @@ def __init__(self, values, dtype=_TD_DTYPE, freq=None, copy=False):
                 "ndarray, or Series or Index containing one of those."
             )
             raise ValueError(msg.format(type(values).__name__))
+        if values.ndim != 1:
+            raise ValueError("Only 1-dimensional input arrays are supported.")
 
         if values.dtype == 'i8':
             # for compat with datetime/timedelta/period shared methods,
@@ -945,6 +947,9 @@ def sequence_to_td64ns(data, copy=False, unit="ns", errors="raise"):
                         .format(dtype=data.dtype))
 
     data = np.array(data, copy=copy)
+    if data.ndim != 1:
+        raise ValueError("Only 1-dimensional input arrays are supported.")
+
     assert data.dtype == 'm8[ns]', data
     return data, inferred_freq
 

pandas/core/indexes/base.py

@@ -665,7 +665,8 @@ def __array_wrap__(self, result, context=None):
         """
         Gets called after a ufunc.
         """
-        if is_bool_dtype(result):
+        result = lib.item_from_zerodim(result)
+        if is_bool_dtype(result) or lib.is_scalar(result):
             return result
 
         attrs = self._get_attributes_dict()

pandas/tests/arithmetic/test_datetime64.py

@@ -1434,6 +1434,20 @@ def test_dt64arr_add_sub_offset_ndarray(self, tz_naive_fixture,
 class TestDatetime64OverflowHandling(object):
     # TODO: box + de-duplicate
 
+    def test_dt64_overflow_masking(self, box_with_array):
+        # GH#25317
+        left = Series([Timestamp('1969-12-31')])
+        right = Series([NaT])
+
+        left = tm.box_expected(left, box_with_array)
+        right = tm.box_expected(right, box_with_array)
+
+        expected = TimedeltaIndex([NaT])
+        expected = tm.box_expected(expected, box_with_array)
+
+        result = left - right
+        tm.assert_equal(result, expected)
+
     def test_dt64_series_arith_overflow(self):
         # GH#12534, fixed by GH#19024
         dt = pd.Timestamp('1700-01-31')

pandas/tests/arrays/test_timedeltas.py

@@ -9,6 +9,18 @@
 
 
 class TestTimedeltaArrayConstructor(object):
+    def test_only_1dim_accepted(self):
+        # GH#25282
+        arr = np.array([0, 1, 2, 3], dtype='m8[h]').astype('m8[ns]')
+
+        with pytest.raises(ValueError, match="Only 1-dimensional"):
+            # 2-dim
+            TimedeltaArray(arr.reshape(2, 2))
+
+        with pytest.raises(ValueError, match="Only 1-dimensional"):
+            # 0-dim
+            TimedeltaArray(arr[[0]].squeeze())
+
     def test_freq_validation(self):
         # ensure that the public constructor cannot create an invalid instance
         arr = np.array([0, 0, 1], dtype=np.int64) * 3600 * 10**9
@@ -51,6 +63,16 @@ def test_copy(self):
 
 
 class TestTimedeltaArray(object):
+    def test_np_sum(self):
+        # GH#25282
+        vals = np.arange(5, dtype=np.int64).view('m8[h]').astype('m8[ns]')
+        arr = TimedeltaArray(vals)
+        result = np.sum(arr)
+        assert result == vals.sum()
+
+        result = np.sum(pd.TimedeltaIndex(arr))
+        assert result == vals.sum()
+
     def test_from_sequence_dtype(self):
         msg = "dtype .*object.* cannot be converted to timedelta64"
         with pytest.raises(ValueError, match=msg):