BUG: fix tz-aware datetime convert to DatetimeIndex (GH 14088)

closes #14088

Author: John Liekezer <cool.Bakov@yandex.ru>

Closes #14090 from conquistador1492/issue_14088 and squashes the following commits:

c91425b [John Liekezer] BUG: fix tz-aware datetime convert to DatetimeIndex (GH 14088)

Author: conquistador1492

doc/source/whatsnew/v0.19.0.txt

@@ -1549,6 +1549,7 @@ Bug Fixes
 - Bug where ``pd.read_gbq()`` could throw ``ImportError: No module named discovery`` as a result of a naming conflict with another python package called apiclient  (:issue:`13454`)
 - Bug in ``Index.union`` returns an incorrect result with a named empty index (:issue:`13432`)
 - Bugs in ``Index.difference`` and ``DataFrame.join`` raise in Python3 when using mixed-integer indexes (:issue:`13432`, :issue:`12814`)
+- Bug in subtract tz-aware ``datetime.datetime`` from tz-aware ``datetime64`` series (:issue:`14088`)
 - Bug in ``.to_excel()`` when DataFrame contains a MultiIndex which contains a label with a NaN value (:issue:`13511`)
 - Bug in invalid frequency offset string like "D1", "-2-3H" may not raise ``ValueError (:issue:`13930`)
 - Bug in ``concat`` and ``groupby`` for hierarchical frames with ``RangeIndex`` levels (:issue:`13542`).

pandas/core/ops.py

@@ -453,7 +453,7 @@ def _convert_to_array(self, values, name=None, other=None):
                 values = values.to_series()
             # datetime with tz
             elif (isinstance(ovalues, datetime.datetime) and
-                  hasattr(ovalues, 'tz')):
+                  hasattr(ovalues, 'tzinfo')):
                 values = pd.DatetimeIndex(values)
             # datetime array with tz
             elif is_datetimetz(values):

pandas/tests/series/test_operators.py

@@ -18,7 +18,8 @@
 
 from pandas.compat import range, zip
 from pandas import compat
-from pandas.util.testing import assert_series_equal, assert_almost_equal
+from pandas.util.testing import (assert_series_equal, assert_almost_equal,
+                                 assert_frame_equal)
 import pandas.util.testing as tm
 
 from .common import TestData
@@ -45,8 +46,8 @@ def test_comparisons(self):
 
         # it works!
         exp = Series([False, False, False])
-        tm.assert_series_equal(s == s2, exp)
-        tm.assert_series_equal(s2 == s, exp)
+        assert_series_equal(s == s2, exp)
+        assert_series_equal(s2 == s, exp)
 
     def test_op_method(self):
         def check(series, other, check_reverse=False):
@@ -64,12 +65,12 @@ def check(series, other, check_reverse=False):
 
                 result = op(series, other)
                 expected = alt(series, other)
-                tm.assert_almost_equal(result, expected)
+                assert_almost_equal(result, expected)
                 if check_reverse:
                     rop = getattr(Series, "r" + opname)
                     result = rop(series, other)
                     expected = alt(other, series)
-                    tm.assert_almost_equal(result, expected)
+                    assert_almost_equal(result, expected)
 
         check(self.ts, self.ts * 2)
         check(self.ts, self.ts[::2])
@@ -149,8 +150,8 @@ def _check_op(series, other, op, pos_only=False,
 
             cython_or_numpy = op(left, right)
             python = left.combine(right, op)
-            tm.assert_series_equal(cython_or_numpy, python,
-                                   check_dtype=check_dtype)
+            assert_series_equal(cython_or_numpy, python,
+                                check_dtype=check_dtype)
 
         def check(series, other):
             simple_ops = ['add', 'sub', 'mul', 'truediv', 'floordiv', 'mod']
@@ -187,7 +188,7 @@ def check_comparators(series, other, check_dtype=True):
     def test_operators_empty_int_corner(self):
         s1 = Series([], [], dtype=np.int32)
         s2 = Series({'x': 0.})
-        tm.assert_series_equal(s1 * s2, Series([np.nan], index=['x']))
+        assert_series_equal(s1 * s2, Series([np.nan], index=['x']))
 
     def test_operators_timedelta64(self):
 
@@ -668,6 +669,16 @@ def run_ops(ops, get_ser, test_ser):
         self.assertRaises(TypeError, lambda: td1 - dt1)
         self.assertRaises(TypeError, lambda: td2 - dt2)
 
+    def test_sub_datetime_compat(self):
+        # GH 14088
+        tm._skip_if_no_pytz()
+        import pytz
+        s = Series([datetime(2016, 8, 23, 12, tzinfo=pytz.utc), pd.NaT])
+        dt = datetime(2016, 8, 22, 12, tzinfo=pytz.utc)
+        exp = Series([Timedelta('1 days'), pd.NaT])
+        assert_series_equal(s - dt, exp)
+        assert_series_equal(s - Timestamp(dt), exp)
+
     def test_sub_single_tz(self):
         # GH12290
         s1 = Series([pd.Timestamp('2016-02-10', tz='America/Sao_Paulo')])
@@ -1175,21 +1186,21 @@ def test_comparison_flex_basic(self):
         left = pd.Series(np.random.randn(10))
         right = pd.Series(np.random.randn(10))
 
-        tm.assert_series_equal(left.eq(right), left == right)
-        tm.assert_series_equal(left.ne(right), left != right)
-        tm.assert_series_equal(left.le(right), left < right)
-        tm.assert_series_equal(left.lt(right), left <= right)
-        tm.assert_series_equal(left.gt(right), left > right)
-        tm.assert_series_equal(left.ge(right), left >= right)
+        assert_series_equal(left.eq(right), left == right)
+        assert_series_equal(left.ne(right), left != right)
+        assert_series_equal(left.le(right), left < right)
+        assert_series_equal(left.lt(right), left <= right)
+        assert_series_equal(left.gt(right), left > right)
+        assert_series_equal(left.ge(right), left >= right)
 
         # axis
         for axis in [0, None, 'index']:
-            tm.assert_series_equal(left.eq(right, axis=axis), left == right)
-            tm.assert_series_equal(left.ne(right, axis=axis), left != right)
-            tm.assert_series_equal(left.le(right, axis=axis), left < right)
-            tm.assert_series_equal(left.lt(right, axis=axis), left <= right)
-            tm.assert_series_equal(left.gt(right, axis=axis), left > right)
-            tm.assert_series_equal(left.ge(right, axis=axis), left >= right)
+            assert_series_equal(left.eq(right, axis=axis), left == right)
+            assert_series_equal(left.ne(right, axis=axis), left != right)
+            assert_series_equal(left.le(right, axis=axis), left < right)
+            assert_series_equal(left.lt(right, axis=axis), left <= right)
+            assert_series_equal(left.gt(right, axis=axis), left > right)
+            assert_series_equal(left.ge(right, axis=axis), left >= right)
 
         #
         msg = 'No axis named 1 for object type'
@@ -1202,44 +1213,44 @@ def test_comparison_flex_alignment(self):
         right = Series([2, 2, 2], index=list('bcd'))
 
         exp = pd.Series([False, False, True, False], index=list('abcd'))
-        tm.assert_series_equal(left.eq(right), exp)
+        assert_series_equal(left.eq(right), exp)
 
         exp = pd.Series([True, True, False, True], index=list('abcd'))
-        tm.assert_series_equal(left.ne(right), exp)
+        assert_series_equal(left.ne(right), exp)
 
         exp = pd.Series([False, False, True, False], index=list('abcd'))
-        tm.assert_series_equal(left.le(right), exp)
+        assert_series_equal(left.le(right), exp)
 
         exp = pd.Series([False, False, False, False], index=list('abcd'))
-        tm.assert_series_equal(left.lt(right), exp)
+        assert_series_equal(left.lt(right), exp)
 
         exp = pd.Series([False, True, True, False], index=list('abcd'))
-        tm.assert_series_equal(left.ge(right), exp)
+        assert_series_equal(left.ge(right), exp)
 
         exp = pd.Series([False, True, False, False], index=list('abcd'))
-        tm.assert_series_equal(left.gt(right), exp)
+        assert_series_equal(left.gt(right), exp)
 
     def test_comparison_flex_alignment_fill(self):
         left = Series([1, 3, 2], index=list('abc'))
         right = Series([2, 2, 2], index=list('bcd'))
 
         exp = pd.Series([False, False, True, True], index=list('abcd'))
-        tm.assert_series_equal(left.eq(right, fill_value=2), exp)
+        assert_series_equal(left.eq(right, fill_value=2), exp)
 
         exp = pd.Series([True, True, False, False], index=list('abcd'))
-        tm.assert_series_equal(left.ne(right, fill_value=2), exp)
+        assert_series_equal(left.ne(right, fill_value=2), exp)
 
         exp = pd.Series([False, False, True, True], index=list('abcd'))
-        tm.assert_series_equal(left.le(right, fill_value=0), exp)
+        assert_series_equal(left.le(right, fill_value=0), exp)
 
         exp = pd.Series([False, False, False, True], index=list('abcd'))
-        tm.assert_series_equal(left.lt(right, fill_value=0), exp)
+        assert_series_equal(left.lt(right, fill_value=0), exp)
 
         exp = pd.Series([True, True, True, False], index=list('abcd'))
-        tm.assert_series_equal(left.ge(right, fill_value=0), exp)
+        assert_series_equal(left.ge(right, fill_value=0), exp)
 
         exp = pd.Series([True, True, False, False], index=list('abcd'))
-        tm.assert_series_equal(left.gt(right, fill_value=0), exp)
+        assert_series_equal(left.gt(right, fill_value=0), exp)
 
     def test_operators_bitwise(self):
         # GH 9016: support bitwise op for integer types
@@ -1426,27 +1437,27 @@ def test_arith_ops_df_compat(self):
 
         exp = pd.Series([3.0, 4.0, np.nan, np.nan],
                         index=list('ABCD'), name='x')
-        tm.assert_series_equal(s1 + s2, exp)
-        tm.assert_series_equal(s2 + s1, exp)
+        assert_series_equal(s1 + s2, exp)
+        assert_series_equal(s2 + s1, exp)
 
         exp = pd.DataFrame({'x': [3.0, 4.0, np.nan, np.nan]},
                            index=list('ABCD'))
-        tm.assert_frame_equal(s1.to_frame() + s2.to_frame(), exp)
-        tm.assert_frame_equal(s2.to_frame() + s1.to_frame(), exp)
+        assert_frame_equal(s1.to_frame() + s2.to_frame(), exp)
+        assert_frame_equal(s2.to_frame() + s1.to_frame(), exp)
 
         # different length
         s3 = pd.Series([1, 2, 3], index=list('ABC'), name='x')
         s4 = pd.Series([2, 2, 2, 2], index=list('ABCD'), name='x')
 
         exp = pd.Series([3, 4, 5, np.nan],
                         index=list('ABCD'), name='x')
-        tm.assert_series_equal(s3 + s4, exp)
-        tm.assert_series_equal(s4 + s3, exp)
+        assert_series_equal(s3 + s4, exp)
+        assert_series_equal(s4 + s3, exp)
 
         exp = pd.DataFrame({'x': [3, 4, 5, np.nan]},
                            index=list('ABCD'))
-        tm.assert_frame_equal(s3.to_frame() + s4.to_frame(), exp)
-        tm.assert_frame_equal(s4.to_frame() + s3.to_frame(), exp)
+        assert_frame_equal(s3.to_frame() + s4.to_frame(), exp)
+        assert_frame_equal(s4.to_frame() + s3.to_frame(), exp)
 
     def test_comp_ops_df_compat(self):
         # GH 1134
@@ -1485,56 +1496,56 @@ def test_bool_ops_df_compat(self):
 
         exp = pd.Series([True, False, False, False],
                         index=list('ABCD'), name='x')
-        tm.assert_series_equal(s1 & s2, exp)
-        tm.assert_series_equal(s2 & s1, exp)
+        assert_series_equal(s1 & s2, exp)
+        assert_series_equal(s2 & s1, exp)
 
         # True | np.nan => True
         exp = pd.Series([True, True, True, False],
                         index=list('ABCD'), name='x')
-        tm.assert_series_equal(s1 | s2, exp)
+        assert_series_equal(s1 | s2, exp)
         # np.nan | True => np.nan, filled with False
         exp = pd.Series([True, True, False, False],
                         index=list('ABCD'), name='x')
-        tm.assert_series_equal(s2 | s1, exp)
+        assert_series_equal(s2 | s1, exp)
 
         # DataFrame doesn't fill nan with False
         exp = pd.DataFrame({'x': [True, False, np.nan, np.nan]},
                            index=list('ABCD'))
-        tm.assert_frame_equal(s1.to_frame() & s2.to_frame(), exp)
-        tm.assert_frame_equal(s2.to_frame() & s1.to_frame(), exp)
+        assert_frame_equal(s1.to_frame() & s2.to_frame(), exp)
+        assert_frame_equal(s2.to_frame() & s1.to_frame(), exp)
 
         exp = pd.DataFrame({'x': [True, True, np.nan, np.nan]},
                            index=list('ABCD'))
-        tm.assert_frame_equal(s1.to_frame() | s2.to_frame(), exp)
-        tm.assert_frame_equal(s2.to_frame() | s1.to_frame(), exp)
+        assert_frame_equal(s1.to_frame() | s2.to_frame(), exp)
+        assert_frame_equal(s2.to_frame() | s1.to_frame(), exp)
 
         # different length
         s3 = pd.Series([True, False, True], index=list('ABC'), name='x')
         s4 = pd.Series([True, True, True, True], index=list('ABCD'), name='x')
 
         exp = pd.Series([True, False, True, False],
                         index=list('ABCD'), name='x')
-        tm.assert_series_equal(s3 & s4, exp)
-        tm.assert_series_equal(s4 & s3, exp)
+        assert_series_equal(s3 & s4, exp)
+        assert_series_equal(s4 & s3, exp)
 
         # np.nan | True => np.nan, filled with False
         exp = pd.Series([True, True, True, False],
                         index=list('ABCD'), name='x')
-        tm.assert_series_equal(s3 | s4, exp)
+        assert_series_equal(s3 | s4, exp)
         # True | np.nan => True
         exp = pd.Series([True, True, True, True],
                         index=list('ABCD'), name='x')
-        tm.assert_series_equal(s4 | s3, exp)
+        assert_series_equal(s4 | s3, exp)
 
         exp = pd.DataFrame({'x': [True, False, True, np.nan]},
                            index=list('ABCD'))
-        tm.assert_frame_equal(s3.to_frame() & s4.to_frame(), exp)
-        tm.assert_frame_equal(s4.to_frame() & s3.to_frame(), exp)
+        assert_frame_equal(s3.to_frame() & s4.to_frame(), exp)
+        assert_frame_equal(s4.to_frame() & s3.to_frame(), exp)
 
         exp = pd.DataFrame({'x': [True, True, True, np.nan]},
                            index=list('ABCD'))
-        tm.assert_frame_equal(s3.to_frame() | s4.to_frame(), exp)
-        tm.assert_frame_equal(s4.to_frame() | s3.to_frame(), exp)
+        assert_frame_equal(s3.to_frame() | s4.to_frame(), exp)
+        assert_frame_equal(s4.to_frame() | s3.to_frame(), exp)
 
     def test_series_frame_radd_bug(self):
         # GH 353
@@ -1546,7 +1557,7 @@ def test_series_frame_radd_bug(self):
         frame = DataFrame({'vals': vals})
         result = 'foo_' + frame
         expected = DataFrame({'vals': vals.map(lambda x: 'foo_' + x)})
-        tm.assert_frame_equal(result, expected)
+        assert_frame_equal(result, expected)
 
         # really raise this time
         with tm.assertRaises(TypeError):
@@ -1571,26 +1582,26 @@ def test_series_radd_more(self):
         for dtype in [None, object]:
             res = 1 + pd.Series([1, 2, 3], dtype=dtype)
             exp = pd.Series([2, 3, 4], dtype=dtype)
-            tm.assert_series_equal(res, exp)
+            assert_series_equal(res, exp)
             res = pd.Series([1, 2, 3], dtype=dtype) + 1
-            tm.assert_series_equal(res, exp)
+            assert_series_equal(res, exp)
 
             res = np.nan + pd.Series([1, 2, 3], dtype=dtype)
             exp = pd.Series([np.nan, np.nan, np.nan], dtype=dtype)
-            tm.assert_series_equal(res, exp)
+            assert_series_equal(res, exp)
             res = pd.Series([1, 2, 3], dtype=dtype) + np.nan
-            tm.assert_series_equal(res, exp)
+            assert_series_equal(res, exp)
 
             s = pd.Series([pd.Timedelta('1 days'), pd.Timedelta('2 days'),
                            pd.Timedelta('3 days')], dtype=dtype)
             exp = pd.Series([pd.Timedelta('4 days'), pd.Timedelta('5 days'),
                              pd.Timedelta('6 days')])
-            tm.assert_series_equal(pd.Timedelta('3 days') + s, exp)
-            tm.assert_series_equal(s + pd.Timedelta('3 days'), exp)
+            assert_series_equal(pd.Timedelta('3 days') + s, exp)
+            assert_series_equal(s + pd.Timedelta('3 days'), exp)
 
         s = pd.Series(['x', np.nan, 'x'])
-        tm.assert_series_equal('a' + s, pd.Series(['ax', np.nan, 'ax']))
-        tm.assert_series_equal(s + 'a', pd.Series(['xa', np.nan, 'xa']))
+        assert_series_equal('a' + s, pd.Series(['ax', np.nan, 'ax']))
+        assert_series_equal(s + 'a', pd.Series(['xa', np.nan, 'xa']))
 
     def test_frame_radd_more(self):
         data = [[1, 2, 3],
@@ -1608,32 +1619,32 @@ def test_frame_radd_more(self):
         for dtype in [None, object]:
             res = 1 + pd.DataFrame([1, 2, 3], dtype=dtype)
             exp = pd.DataFrame([2, 3, 4], dtype=dtype)
-            tm.assert_frame_equal(res, exp)
+            assert_frame_equal(res, exp)
             res = pd.DataFrame([1, 2, 3], dtype=dtype) + 1
-            tm.assert_frame_equal(res, exp)
+            assert_frame_equal(res, exp)
 
             res = np.nan + pd.DataFrame([1, 2, 3], dtype=dtype)
             exp = pd.DataFrame([np.nan, np.nan, np.nan], dtype=dtype)
-            tm.assert_frame_equal(res, exp)
+            assert_frame_equal(res, exp)
             res = pd.DataFrame([1, 2, 3], dtype=dtype) + np.nan
-            tm.assert_frame_equal(res, exp)
+            assert_frame_equal(res, exp)
 
         df = pd.DataFrame(['x', np.nan, 'x'])
-        tm.assert_frame_equal('a' + df, pd.DataFrame(['ax', np.nan, 'ax']))
-        tm.assert_frame_equal(df + 'a', pd.DataFrame(['xa', np.nan, 'xa']))
+        assert_frame_equal('a' + df, pd.DataFrame(['ax', np.nan, 'ax']))
+        assert_frame_equal(df + 'a', pd.DataFrame(['xa', np.nan, 'xa']))
 
     def test_operators_frame(self):
         # rpow does not work with DataFrame
         df = DataFrame({'A': self.ts})
 
-        tm.assert_series_equal(self.ts + self.ts, self.ts + df['A'],
-                               check_names=False)
-        tm.assert_series_equal(self.ts ** self.ts, self.ts ** df['A'],
-                               check_names=False)
-        tm.assert_series_equal(self.ts < self.ts, self.ts < df['A'],
-                               check_names=False)
-        tm.assert_series_equal(self.ts / self.ts, self.ts / df['A'],
-                               check_names=False)
+        assert_series_equal(self.ts + self.ts, self.ts + df['A'],
+                            check_names=False)
+        assert_series_equal(self.ts ** self.ts, self.ts ** df['A'],
+                            check_names=False)
+        assert_series_equal(self.ts < self.ts, self.ts < df['A'],
+                            check_names=False)
+        assert_series_equal(self.ts / self.ts, self.ts / df['A'],
+                            check_names=False)
 
     def test_operators_combine(self):
         def _check_fill(meth, op, a, b, fill_value=0):
@@ -1729,12 +1740,12 @@ def test_divide_decimal(self):
         s = Series([Decimal(10)])
         s = s / Decimal(2)
 
-        tm.assert_series_equal(expected, s)
+        assert_series_equal(expected, s)
 
         s = Series([Decimal(10)])
         s = s // Decimal(2)
 
-        tm.assert_series_equal(expected, s)
+        assert_series_equal(expected, s)
 
     def test_datetime64_with_index(self):