Merge branch 'master' into reindex_tzaware

Author: kandersolar

doc/source/whatsnew/v1.0.1.rst

@@ -15,7 +15,7 @@ including other versions of pandas.
 
 Bug fixes
 ~~~~~~~~~
-
+- Bug in :meth:`GroupBy.apply` was raising ``TypeError`` if called with function which returned a non-pandas non-scalar object (e.g. a list) (:issue:`31441`)
 
 Categorical
 ^^^^^^^^^^^
@@ -25,8 +25,8 @@ Categorical
 
 Datetimelike
 ^^^^^^^^^^^^
--
--
+- Fixed regression in :meth:`to_datetime` when parsing non-nanosecond resolution datetimes (:issue:`31491`)
+- Fixed bug in :meth:`to_datetime` raising when ``cache=True`` and out-of-bound values are present (:issue:`31491`)
 
 Timedelta
 ^^^^^^^^^
@@ -44,6 +44,7 @@ Timezones
 Numeric
 ^^^^^^^
 - Bug in dtypes being lost in ``DataFrame.__invert__`` (``~`` operator) with mixed dtypes (:issue:`31183`)
+- Bug in :class:`Series` multiplication when multiplying a numeric :class:`Series` with >10000 elements with a timedelta-like scalar (:issue:`31467`)
 -
 
 Conversion
@@ -68,8 +69,11 @@ Interval
 Indexing
 ^^^^^^^^
 
+- Fixed regression when indexing a ``Series`` or ``DataFrame`` indexed by ``DatetimeIndex`` with a slice containg a :class:`datetime.date` (:issue:`31501`)
+- Fixed regression in :class:`DataFrame` setting values with a slice (e.g. ``df[-4:] = 1``) indexing by label instead of position (:issue:`31469`)
 -
 -
+- Bug where assigning to a :class:`Series` using a IntegerArray / BooleanArray as a mask would raise ``TypeError`` (:issue:`31446`)
 
 Missing
 ^^^^^^^
@@ -86,6 +90,7 @@ MultiIndex
 I/O
 ^^^
 
+- Fixed regression in :meth:`~DataFrame.to_csv` where specifying an ``na_rep`` might truncate the values written (:issue:`31447`)
 -
 -
 

doc/source/whatsnew/v1.1.0.rst

@@ -150,7 +150,8 @@ Indexing
 - Bug in :meth:`PeriodIndex.get_loc` treating higher-resolution strings differently from :meth:`PeriodIndex.get_value` (:issue:`31172`)
 - Bug in :meth:`Series.at` and :meth:`DataFrame.at` not matching ``.loc`` behavior when looking up an integer in a :class:`Float64Index` (:issue:`31329`)
 - Bug in :meth:`PeriodIndex.is_monotonic` incorrectly returning ``True`` when containing leading ``NaT`` entries (:issue:`31437`)
-- Bug in :meth:`DataFrame.reindex` and :meth:`Series.reindex` when reindexing with a tz-aware index (:issue:`26683`)
+- Bug in :meth:`DatetimeIndex.get_loc` raising ``KeyError`` with converted-integer key instead of the user-passed key (:issue:`31425`)
+- Bug in :meth:`DataFrame.reindex` and :meth:`Series.reindex` when reindexing with a tz-aware index and method='nearest' (:issue:`26683`)
 
 Missing
 ^^^^^^^

pandas/_libs/reduction.pyx

@@ -501,9 +501,9 @@ def apply_frame_axis0(object frame, object f, object names,
 
             if not is_scalar(piece):
                 # Need to copy data to avoid appending references
-                if hasattr(piece, "copy"):
+                try:
                     piece = piece.copy(deep="all")
-                else:
+                except (TypeError, AttributeError):
                     piece = copy(piece)
 
             results.append(piece)

pandas/_libs/tslibs/period.pyx

@@ -473,9 +473,6 @@ cdef int DtoQ_yq(int64_t ordinal, asfreq_info *af_info, int *year) nogil:
         int quarter
 
     pandas_datetime_to_datetimestruct(ordinal, NPY_FR_D, &dts)
-    # TODO: Another version of this function used
-    # date_info_from_days_and_time(&dts, unix_date, 0)
-    # instead of pandas_datetime_to_datetimestruct; is one more performant?
     if af_info.to_end != 12:
         dts.month -= af_info.to_end
         if dts.month <= 0:
@@ -516,7 +513,7 @@ cdef int64_t asfreq_DTtoW(int64_t ordinal, asfreq_info *af_info) nogil:
 # Conversion _from_ BusinessDay Freq
 
 cdef int64_t asfreq_BtoDT(int64_t ordinal, asfreq_info *af_info) nogil:
-    ordinal = ((ordinal + 3) // 5) * 7 + (ordinal + 3) % 5 -3
+    ordinal = ((ordinal + 3) // 5) * 7 + (ordinal + 3) % 5 - 3
     return upsample_daytime(ordinal, af_info)
 
 
@@ -753,14 +750,7 @@ cdef int64_t get_period_ordinal(npy_datetimestruct *dts, int freq) nogil:
         if fmonth == 0:
             fmonth = 12
 
-        mdiff = dts.month - fmonth
-        # TODO: Aren't the next two conditions equivalent to
-        # unconditional incrementing?
-        if mdiff < 0:
-            mdiff += 12
-        if dts.month >= fmonth:
-            mdiff += 12
-
+        mdiff = dts.month - fmonth + 12
         return (dts.year - 1970) * 4 + (mdiff - 1) // 3
 
     elif freq == FR_MTH:
@@ -804,17 +794,16 @@ cdef int64_t get_period_ordinal(npy_datetimestruct *dts, int freq) nogil:
         delta = (unix_date + 3) % 7 + 1
         # return the number of business days in full weeks plus the business
         # days in the last - possible partial - week
-        if delta <= 5:
-            return (5 * weeks) + delta - 4
-        else:
-            return (5 * weeks) + (5 + 1) - 4
+        if delta > 6:
+            # We have a Sunday, which rolls back to the previous Friday,
+            # just like Saturday, so decrement delta by 1 to treat as saturday
+            delta = 6
+        return (5 * weeks) + delta - 4
 
     elif freq_group == FR_WK:
         day_adj = freq - FR_WK
         return (unix_date + 3 - day_adj) // 7 + 1
 
-    # raise ValueError
-
 
 cdef void get_date_info(int64_t ordinal, int freq,
                         npy_datetimestruct *dts) nogil:
@@ -983,7 +972,7 @@ cdef inline int month_to_quarter(int month) nogil:
 
 @cython.wraparound(False)
 @cython.boundscheck(False)
-def dt64arr_to_periodarr(int64_t[:] dtarr, int freq, tz=None):
+def dt64arr_to_periodarr(const int64_t[:] dtarr, int freq, tz=None):
     """
     Convert array of datetime64 values (passed in as 'i8' dtype) to a set of
     periods corresponding to desired frequency, per period convention.
@@ -1383,7 +1372,7 @@ cdef int pdays_in_month(int64_t ordinal, int freq):
 
 @cython.wraparound(False)
 @cython.boundscheck(False)
-def get_period_field_arr(int code, int64_t[:] arr, int freq):
+def get_period_field_arr(int code, const int64_t[:] arr, int freq):
     cdef:
         Py_ssize_t i, sz
         int64_t[:] out
@@ -1496,7 +1485,7 @@ def extract_freq(ndarray[object] values):
 
 @cython.wraparound(False)
 @cython.boundscheck(False)
-cdef int64_t[:] localize_dt64arr_to_period(int64_t[:] stamps,
+cdef int64_t[:] localize_dt64arr_to_period(const int64_t[:] stamps,
                                            int freq, object tz):
     cdef:
         Py_ssize_t n = len(stamps)
@@ -1584,7 +1573,7 @@ cdef class _Period:
         return freq
 
     @classmethod
-    def _from_ordinal(cls, ordinal, freq):
+    def _from_ordinal(cls, ordinal: int, freq) -> "Period":
         """
         Fast creation from an ordinal and freq that are already validated!
         """
@@ -1704,7 +1693,7 @@ cdef class _Period:
         else:
             return NotImplemented
 
-    def asfreq(self, freq, how='E'):
+    def asfreq(self, freq, how='E') -> "Period":
         """
         Convert Period to desired frequency, at the start or end of the interval.
 
@@ -1735,7 +1724,7 @@ cdef class _Period:
         return Period(ordinal=ordinal, freq=freq)
 
     @property
-    def start_time(self):
+    def start_time(self) -> Timestamp:
         """
         Get the Timestamp for the start of the period.
 
@@ -1765,13 +1754,13 @@ cdef class _Period:
         return self.to_timestamp(how='S')
 
     @property
-    def end_time(self):
+    def end_time(self) -> Timestamp:
         # freq.n can't be negative or 0
         # ordinal = (self + self.freq.n).start_time.value - 1
         ordinal = (self + self.freq).start_time.value - 1
         return Timestamp(ordinal)
 
-    def to_timestamp(self, freq=None, how='start', tz=None):
+    def to_timestamp(self, freq=None, how='start', tz=None) -> Timestamp:
         """
         Return the Timestamp representation of the Period.
 
@@ -1811,17 +1800,17 @@ cdef class _Period:
         return Timestamp(dt64, tz=tz)
 
     @property
-    def year(self):
+    def year(self) -> int:
         base, mult = get_freq_code(self.freq)
         return pyear(self.ordinal, base)
 
     @property
-    def month(self):
+    def month(self) -> int:
         base, mult = get_freq_code(self.freq)
         return pmonth(self.ordinal, base)
 
     @property
-    def day(self):
+    def day(self) -> int:
         """
         Get day of the month that a Period falls on.
 
@@ -1844,7 +1833,7 @@ cdef class _Period:
         return pday(self.ordinal, base)
 
     @property
-    def hour(self):
+    def hour(self) -> int:
         """
         Get the hour of the day component of the Period.
 
@@ -1874,7 +1863,7 @@ cdef class _Period:
         return phour(self.ordinal, base)
 
     @property
-    def minute(self):
+    def minute(self) -> int:
         """
         Get minute of the hour component of the Period.
 
@@ -1898,7 +1887,7 @@ cdef class _Period:
         return pminute(self.ordinal, base)
 
     @property
-    def second(self):
+    def second(self) -> int:
         """
         Get the second component of the Period.
 
@@ -1922,12 +1911,12 @@ cdef class _Period:
         return psecond(self.ordinal, base)
 
     @property
-    def weekofyear(self):
+    def weekofyear(self) -> int:
         base, mult = get_freq_code(self.freq)
         return pweek(self.ordinal, base)
 
     @property
-    def week(self):
+    def week(self) -> int:
         """
         Get the week of the year on the given Period.
 
@@ -1957,7 +1946,7 @@ cdef class _Period:
         return self.weekofyear
 
     @property
-    def dayofweek(self):
+    def dayofweek(self) -> int:
         """
         Day of the week the period lies in, with Monday=0 and Sunday=6.
 
@@ -2008,7 +1997,7 @@ cdef class _Period:
         return pweekday(self.ordinal, base)
 
     @property
-    def weekday(self):
+    def weekday(self) -> int:
         """
         Day of the week the period lies in, with Monday=0 and Sunday=6.
 
@@ -2061,7 +2050,7 @@ cdef class _Period:
         return self.dayofweek
 
     @property
-    def dayofyear(self):
+    def dayofyear(self) -> int:
         """
         Return the day of the year.
 
@@ -2096,12 +2085,12 @@ cdef class _Period:
         return pday_of_year(self.ordinal, base)
 
     @property
-    def quarter(self):
+    def quarter(self) -> int:
         base, mult = get_freq_code(self.freq)
         return pquarter(self.ordinal, base)
 
     @property
-    def qyear(self):
+    def qyear(self) -> int:
         """
         Fiscal year the Period lies in according to its starting-quarter.
 
@@ -2145,7 +2134,7 @@ cdef class _Period:
         return pqyear(self.ordinal, base)
 
     @property
-    def days_in_month(self):
+    def days_in_month(self) -> int:
         """
         Get the total number of days in the month that this period falls on.
 
@@ -2179,7 +2168,7 @@ cdef class _Period:
         return pdays_in_month(self.ordinal, base)
 
     @property
-    def daysinmonth(self):
+    def daysinmonth(self) -> int:
         """
         Get the total number of days of the month that the Period falls in.
 
@@ -2209,7 +2198,7 @@ cdef class _Period:
         return Period(datetime.now(), freq=freq)
 
     @property
-    def freqstr(self):
+    def freqstr(self) -> str:
         return self.freq.freqstr
 
     def __repr__(self) -> str:

pandas/conftest.py

@@ -868,6 +868,16 @@ def tick_classes(request):
     )
 
 
+@pytest.fixture
+def datetime_series():
+    """
+    Fixture for Series of floats with DatetimeIndex
+    """
+    s = tm.makeTimeSeries()
+    s.name = "ts"
+    return s
+
+
 @pytest.fixture
 def float_frame():
     """

pandas/core/algorithms.py

@@ -29,6 +29,7 @@
     is_categorical_dtype,
     is_complex_dtype,
     is_datetime64_any_dtype,
+    is_datetime64_dtype,
     is_datetime64_ns_dtype,
     is_extension_array_dtype,
     is_float_dtype,
@@ -191,6 +192,11 @@ def _reconstruct_data(values, dtype, original):
         if isinstance(original, ABCIndexClass):
             values = values.astype(object, copy=False)
     elif dtype is not None:
+        if is_datetime64_dtype(dtype):
+            dtype = "datetime64[ns]"
+        elif is_timedelta64_dtype(dtype):
+            dtype = "timedelta64[ns]"
+
         values = values.astype(dtype, copy=False)
 
     return values

pandas/core/arrays/boolean.py

@@ -26,6 +26,7 @@
 from pandas.core.dtypes.missing import isna, notna
 
 from pandas.core import nanops, ops
+from pandas.core.indexers import check_array_indexer
 
 from .masked import BaseMaskedArray
 
@@ -382,6 +383,7 @@ def __setitem__(self, key, value):
             value = value[0]
             mask = mask[0]
 
+        key = check_array_indexer(self, key)
         self._data[key] = value
         self._mask[key] = mask
 

pandas/core/arrays/categorical.py

@@ -2073,6 +2073,8 @@ def __setitem__(self, key, value):
 
         lindexer = self.categories.get_indexer(rvalue)
         lindexer = self._maybe_coerce_indexer(lindexer)
+
+        key = check_array_indexer(self, key)
         self._codes[key] = lindexer
 
     def _reverse_indexer(self) -> Dict[Hashable, np.ndarray]: