Merge remote-tracking branch 'upstream/master' into issue20825

Author: Dr-Irv

ci/travis-36.yaml

@@ -18,12 +18,10 @@ dependencies:
   - numexpr
   - numpy
   - openpyxl
-  - pandas-datareader
   - psycopg2
   - pyarrow
   - pymysql
   - pytables
-  - python-dateutil
   - python-snappy
   - python=3.6*
   - pytz
@@ -45,3 +43,5 @@ dependencies:
   - pip:
     - brotlipy
     - coverage
+    - pandas-datareader
+    - python-dateutil

doc/source/whatsnew/v0.23.1.txt

@@ -52,6 +52,7 @@ Groupby/Resample/Rolling
 ^^^^^^^^^^^^^^^^^^^^^^^^
 
 - Bug in :func:`DataFrame.agg` where applying multiple aggregation functions to a :class:`DataFrame` with duplicated column names would cause a stack overflow (:issue:`21063`)
+- Bug in :func:`pandas.core.groupby.GroupBy.ffill` and :func:`pandas.core.groupby.GroupBy.bfill` where the fill within a grouping would not always be applied as intended due to the implementations' use of a non-stable sort (:issue:`21207`)
 
 Strings
 ^^^^^^^
@@ -66,6 +67,7 @@ Categorical
 ^^^^^^^^^^^
 
 - Bug in :func:`pandas.util.testing.assert_index_equal` which raised ``AssertionError`` incorrectly, when comparing two :class:`CategoricalIndex` objects with param ``check_categorical=False`` (:issue:`19776`)
+- Bug in :meth:`Categorical.fillna` incorrectly raising a ``TypeError`` when `value` the individual categories are iterable and `value` is an iterable (:issue:`21097`, :issue:`19788`)
 
 Conversion
 ^^^^^^^^^^
@@ -78,12 +80,14 @@ Indexing
 
 - Bug in :meth:`Series.reset_index` where appropriate error was not raised with an invalid level name (:issue:`20925`)
 - Bug in :func:`interval_range` when ``start``/``periods`` or ``end``/``periods`` are specified with float ``start`` or ``end`` (:issue:`21161`)
+- Bug in :meth:`MultiIndex.set_names` where error raised for a ``MultiIndex`` with ``nlevels == 1`` (:issue:`21149`)
 -
 
 I/O
 ^^^
 
--
+- Bug in IO methods specifying ``compression='zip'`` which produced uncompressed zip archives (:issue:`17778`, :issue:`21144`)
+- Bug in :meth:`DataFrame.to_stata` which prevented exporting DataFrames to buffers and most file-like objects (:issue:`21041`)
 -
 
 Plotting
@@ -97,8 +101,3 @@ Reshaping
 
 - Bug in :func:`concat` where error was raised in concatenating :class:`Series` with numpy scalar and tuple names (:issue:`21015`)
 -
-
-Categorical
-^^^^^^^^^^^
-
--

doc/source/whatsnew/v0.24.0.txt

@@ -1,7 +1,7 @@
 .. _whatsnew_0240:
 
 v0.24.0
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+-------
 
 .. _whatsnew_0240.enhancements:
 
@@ -12,7 +12,7 @@ New features
 
 Other Enhancements
 ^^^^^^^^^^^^^^^^^^
--
+- :func:`to_datetime` now supports the ``%Z`` and ``%z`` directive when passed into ``format`` (:issue:`13486`)
 -
 -
 
@@ -22,6 +22,13 @@ Other Enhancements
 Backwards incompatible API changes
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
+.. _whatsnew_0240.api.datetimelike:
+
+Datetimelike API Changes
+^^^^^^^^^^^^^^^^^^^^^^^^
+
+- For :class:`DatetimeIndex` and :class:`TimedeltaIndex` with non-``None`` ``freq`` attribute, addition or subtraction of integer-dtyped array or ``Index`` will return an object of the same class (:issue:`19959`)
+
 .. _whatsnew_0240.api.other:
 
 Other API Changes

pandas/_libs/groupby.pyx

@@ -297,7 +297,8 @@ def group_fillna_indexer(ndarray[int64_t] out, ndarray[int64_t] labels,
     # Make sure all arrays are the same size
     assert N == len(labels) == len(mask)
 
-    sorted_labels = np.argsort(labels).astype(np.int64, copy=False)
+    sorted_labels = np.argsort(labels, kind='mergesort').astype(
+        np.int64, copy=False)
     if direction == 'bfill':
         sorted_labels = sorted_labels[::-1]
 

pandas/_libs/tslibs/strptime.pyx

@@ -20,6 +20,7 @@ except:
         except:
             from _dummy_thread import allocate_lock as _thread_allocate_lock
 
+import pytz
 
 from cython cimport Py_ssize_t
 from cpython cimport PyFloat_Check
@@ -40,6 +41,27 @@ from util cimport is_string_object
 from nattype cimport checknull_with_nat, NPY_NAT
 from nattype import nat_strings
 
+cdef dict _parse_code_table = {'y': 0,
+                               'Y': 1,
+                               'm': 2,
+                               'B': 3,
+                               'b': 4,
+                               'd': 5,
+                               'H': 6,
+                               'I': 7,
+                               'M': 8,
+                               'S': 9,
+                               'f': 10,
+                               'A': 11,
+                               'a': 12,
+                               'w': 13,
+                               'j': 14,
+                               'U': 15,
+                               'W': 16,
+                               'Z': 17,
+                               'p': 18,  # an additional key, only with I
+                               'z': 19}
+
 
 def array_strptime(ndarray[object] values, object fmt,
                    bint exact=True, errors='raise'):
@@ -58,15 +80,15 @@ def array_strptime(ndarray[object] values, object fmt,
         Py_ssize_t i, n = len(values)
         pandas_datetimestruct dts
         ndarray[int64_t] iresult
-        int year, month, day, minute, hour, second, weekday, julian, tz
-        int week_of_year, week_of_year_start
+        ndarray[object] result_timezone
+        int year, month, day, minute, hour, second, weekday, julian
+        int week_of_year, week_of_year_start, parse_code, ordinal
         int64_t us, ns
-        object val, group_key, ampm, found
+        object val, group_key, ampm, found, timezone
         dict found_key
         bint is_raise = errors=='raise'
         bint is_ignore = errors=='ignore'
         bint is_coerce = errors=='coerce'
-        int ordinal
 
     assert is_raise or is_ignore or is_coerce
 
@@ -79,6 +101,8 @@ def array_strptime(ndarray[object] values, object fmt,
                     in fmt):
                 raise ValueError("Cannot use '%W' or '%U' without "
                                  "day and year")
+        elif '%Z' in fmt and '%z' in fmt:
+            raise ValueError("Cannot parse both %Z and %z")
 
     global _TimeRE_cache, _regex_cache
     with _cache_lock:
@@ -108,32 +132,10 @@ def array_strptime(ndarray[object] values, object fmt,
 
     result = np.empty(n, dtype='M8[ns]')
     iresult = result.view('i8')
+    result_timezone = np.empty(n, dtype='object')
 
     dts.us = dts.ps = dts.as = 0
 
-    cdef dict _parse_code_table = {
-        'y': 0,
-        'Y': 1,
-        'm': 2,
-        'B': 3,
-        'b': 4,
-        'd': 5,
-        'H': 6,
-        'I': 7,
-        'M': 8,
-        'S': 9,
-        'f': 10,
-        'A': 11,
-        'a': 12,
-        'w': 13,
-        'j': 14,
-        'U': 15,
-        'W': 16,
-        'Z': 17,
-        'p': 18   # just an additional key, works only with I
-    }
-    cdef int parse_code
-
     for i in range(n):
         val = values[i]
         if is_string_object(val):
@@ -176,7 +178,7 @@ def array_strptime(ndarray[object] values, object fmt,
         year = 1900
         month = day = 1
         hour = minute = second = ns = us = 0
-        tz = -1
+        timezone = None
         # Default to -1 to signify that values not known; not critical to have,
         # though
         week_of_year = -1
@@ -266,21 +268,10 @@ def array_strptime(ndarray[object] values, object fmt,
                     # W starts week on Monday.
                     week_of_year_start = 0
             elif parse_code == 17:
-                # Since -1 is default value only need to worry about setting tz
-                # if it can be something other than -1.
-                found_zone = found_dict['Z'].lower()
-                for value, tz_values in enumerate(locale_time.timezone):
-                    if found_zone in tz_values:
-                        # Deal w/ bad locale setup where timezone names are the
-                        # same and yet time.daylight is true; too ambiguous to
-                        # be able to tell what timezone has daylight savings
-                        if (time.tzname[0] == time.tzname[1] and
-                            time.daylight and found_zone not in (
-                                "utc", "gmt")):
-                            break
-                        else:
-                            tz = value
-                            break
+                timezone = pytz.timezone(found_dict['Z'])
+            elif parse_code == 19:
+                timezone = parse_timezone_directive(found_dict['z'])
+
         # If we know the wk of the year and what day of that wk, we can figure
         # out the Julian day of the year.
         if julian == -1 and week_of_year != -1 and weekday != -1:
@@ -330,7 +321,9 @@ def array_strptime(ndarray[object] values, object fmt,
                 continue
             raise
 
-    return result
+        result_timezone[i] = timezone
+
+    return result, result_timezone
 
 
 """_getlang, LocaleTime, TimeRE, _calc_julian_from_U_or_W are vendored
@@ -538,14 +531,13 @@ class TimeRE(dict):
             # XXX: Does 'Y' need to worry about having less or more than
             #     4 digits?
             'Y': r"(?P<Y>\d\d\d\d)",
+            'z': r"(?P<z>[+-]\d\d:?[0-5]\d(:?[0-5]\d(\.\d{1,6})?)?|Z)",
             'A': self.__seqToRE(self.locale_time.f_weekday, 'A'),
             'a': self.__seqToRE(self.locale_time.a_weekday, 'a'),
             'B': self.__seqToRE(self.locale_time.f_month[1:], 'B'),
             'b': self.__seqToRE(self.locale_time.a_month[1:], 'b'),
             'p': self.__seqToRE(self.locale_time.am_pm, 'p'),
-            'Z': self.__seqToRE([tz for tz_names in self.locale_time.timezone
-                                 for tz in tz_names],
-                                'Z'),
+            'Z': self.__seqToRE(pytz.all_timezones, 'Z'),
             '%': '%'})
         base.__setitem__('W', base.__getitem__('U').replace('U', 'W'))
         base.__setitem__('c', self.pattern(self.locale_time.LC_date_time))
@@ -632,3 +624,50 @@ cdef _calc_julian_from_U_or_W(int year, int week_of_year,
     else:
         days_to_week = week_0_length + (7 * (week_of_year - 1))
         return 1 + days_to_week + day_of_week
+
+cdef parse_timezone_directive(object z):
+    """
+    Parse the '%z' directive and return a pytz.FixedOffset
+
+    Parameters
+    ----------
+    z : string of the UTC offset
+
+    Returns
+    -------
+    pytz.FixedOffset
+
+    Notes
+    -----
+    This is essentially similar to the cpython implementation
+    https://github.com/python/cpython/blob/master/Lib/_strptime.py#L457-L479
+    """
+
+    cdef:
+        int gmtoff_fraction, hours, minutes, seconds, pad_number, microseconds
+        int total_minutes
+        object gmtoff_remainder, gmtoff_remainder_padding
+
+    if z == 'Z':
+        return pytz.FixedOffset(0)
+    if z[3] == ':':
+        z = z[:3] + z[4:]
+        if len(z) > 5:
+            if z[5] != ':':
+                msg = "Inconsistent use of : in {0}"
+                raise ValueError(msg.format(z))
+            z = z[:5] + z[6:]
+    hours = int(z[1:3])
+    minutes = int(z[3:5])
+    seconds = int(z[5:7] or 0)
+
+    # Pad to always return microseconds.
+    gmtoff_remainder = z[8:]
+    pad_number = 6 - len(gmtoff_remainder)
+    gmtoff_remainder_padding = "0" * pad_number
+    microseconds = int(gmtoff_remainder + gmtoff_remainder_padding)
+
+    total_minutes = ((hours * 60) + minutes + (seconds / 60) +
+                     (microseconds / 60000000))
+    total_minutes = -total_minutes if z.startswith("-") else total_minutes
+    return pytz.FixedOffset(total_minutes)

pandas/core/arrays/categorical.py

@@ -12,6 +12,7 @@
 from pandas.core.dtypes.generic import (
     ABCSeries, ABCIndexClass, ABCCategoricalIndex)
 from pandas.core.dtypes.missing import isna, notna
+from pandas.core.dtypes.inference import is_hashable
 from pandas.core.dtypes.cast import (
     maybe_infer_to_datetimelike,
     coerce_indexer_dtype)
@@ -1751,7 +1752,7 @@ def fillna(self, value=None, method=None, limit=None):
                 values[indexer] = values_codes[values_codes != -1]
 
             # If value is not a dict or Series it should be a scalar
-            elif is_scalar(value):
+            elif is_hashable(value):
                 if not isna(value) and value not in self.categories:
                     raise ValueError("fill value must be in categories")
 

pandas/core/frame.py

@@ -1774,8 +1774,11 @@ def to_stata(self, fname, convert_dates=None, write_index=True,
 
         Parameters
         ----------
-        fname : str or buffer
-            String path of file-like object.
+        fname : path (string), buffer or path object
+            string, path object (pathlib.Path or py._path.local.LocalPath) or
+            object implementing a binary write() functions. If using a buffer
+            then the buffer will not be automatically closed after the file
+            data has been written.
         convert_dates : dict
             Dictionary mapping columns containing datetime types to stata
             internal format to use when writing the dates. Options are 'tc',
@@ -4174,8 +4177,9 @@ def dropna(self, axis=0, how='any', thresh=None, subset=None,
             * 0, or 'index' : Drop rows which contain missing values.
             * 1, or 'columns' : Drop columns which contain missing value.
 
-            .. deprecated:: 0.23.0: Pass tuple or list to drop on multiple
-            axes.
+            .. deprecated:: 0.23.0
+                Pass tuple or list to drop on multiple axes.
+
         how : {'any', 'all'}, default 'any'
             Determine if row or column is removed from DataFrame, when we have
             at least one NA or all NA.
@@ -7085,6 +7089,9 @@ def quantile(self, q=0.5, axis=0, numeric_only=True,
             0 <= q <= 1, the quantile(s) to compute
         axis : {0, 1, 'index', 'columns'} (default 0)
             0 or 'index' for row-wise, 1 or 'columns' for column-wise
+        numeric_only : boolean, default True
+            If False, the quantile of datetime and timedelta data will be
+            computed as well
         interpolation : {'linear', 'lower', 'higher', 'midpoint', 'nearest'}
             .. versionadded:: 0.18.0
 
@@ -7112,7 +7119,7 @@ def quantile(self, q=0.5, axis=0, numeric_only=True,
         --------
 
         >>> df = pd.DataFrame(np.array([[1, 1], [2, 10], [3, 100], [4, 100]]),
-                           columns=['a', 'b'])
+                              columns=['a', 'b'])
         >>> df.quantile(.1)
         a    1.3
         b    3.7
@@ -7122,6 +7129,20 @@ def quantile(self, q=0.5, axis=0, numeric_only=True,
         0.1  1.3   3.7
         0.5  2.5  55.0
 
+        Specifying `numeric_only=False` will also compute the quantile of
+        datetime and timedelta data.
+
+        >>> df = pd.DataFrame({'A': [1, 2],
+                               'B': [pd.Timestamp('2010'),
+                                     pd.Timestamp('2011')],
+                               'C': [pd.Timedelta('1 days'),
+                                     pd.Timedelta('2 days')]})
+        >>> df.quantile(0.5, numeric_only=False)
+        A                    1.5
+        B    2010-07-02 12:00:00
+        C        1 days 12:00:00
+        Name: 0.5, dtype: object
+
         See Also
         --------
         pandas.core.window.Rolling.quantile