Merge branch 'master' of https://github.com/pandas-dev/pandas into tslibs-offsets-years

Author: jbrockmendel

doc/source/io.rst

@@ -4427,8 +4427,10 @@ Several caveats.
 
 - This is a newer library, and the format, though stable, is not guaranteed to be backward compatible
   to the earlier versions.
-- The format will NOT write an ``Index``, or ``MultiIndex`` for the ``DataFrame`` and will raise an
-  error if a non-default one is provided. You can simply ``.reset_index()`` in order to store the index.
+- The format will NOT write an ``Index``, or ``MultiIndex`` for the
+  ``DataFrame`` and will raise an error if a non-default one is provided. You
+  can ``.reset_index()`` to store the index or ``.reset_index(drop=True)`` to
+  ignore it.
 - Duplicate column names and non-string columns names are not supported
 - Non supported types include ``Period`` and actual python object types. These will raise a helpful error message
   on an attempt at serialization.
@@ -4491,8 +4493,10 @@ dtypes, including extension dtypes such as datetime with tz.
 
 Several caveats.
 
-- The format will NOT write an ``Index``, or ``MultiIndex`` for the ``DataFrame`` and will raise an
-  error if a non-default one is provided. You can simply ``.reset_index(drop=True)`` in order to store the index.
+- The format will NOT write an ``Index``, or ``MultiIndex`` for the
+  ``DataFrame`` and will raise an error if a non-default one is provided. You
+  can ``.reset_index()`` to store the index or ``.reset_index(drop=True)`` to
+  ignore it.
 - Duplicate column names and non-string columns names are not supported
 - Categorical dtypes can be serialized to parquet, but will de-serialize as ``object`` dtype.
 - Non supported types include ``Period`` and actual python object types. These will raise a helpful error message
@@ -4538,7 +4542,7 @@ Read from a parquet file.
 
    result.dtypes
 
-Read only certain columns of a parquet file. 
+Read only certain columns of a parquet file.
 
 .. ipython:: python
 

doc/source/whatsnew/v0.21.1.txt

@@ -86,6 +86,7 @@ I/O
 - Bug in :func:`read_csv` for handling null values in index columns when specifying ``na_filter=False`` (:issue:`5239`)
 - Bug in :meth:`DataFrame.to_csv` when the table had ``MultiIndex`` columns, and a list of strings was passed in for ``header`` (:issue:`5539`)
 - :func:`read_parquet` now allows to specify the columns to read from a parquet file (:issue:`18154`)
+- :func:`read_parquet` now allows to specify kwargs which are passed to the respective engine (:issue:`18216`)
 
 Plotting
 ^^^^^^^^

pandas/_libs/lib.pyx

@@ -1484,7 +1484,7 @@ def get_blkno_indexers(int64_t[:] blknos, bint group=True):
             if len(slices) == 1:
                 yield blkno, slice(slices[0][0], slices[0][1])
             else:
-                tot_len = sum([stop - start for start, stop in slices])
+                tot_len = sum(stop - start for start, stop in slices)
                 result = np.empty(tot_len, dtype=np.int64)
                 res_view = result
 

pandas/compat/__init__.py

@@ -266,7 +266,7 @@ def east_asian_len(data, encoding=None, ambiguous_width=1):
         Calculate display width considering unicode East Asian Width
         """
         if isinstance(data, text_type):
-            return sum([_EAW_MAP.get(east_asian_width(c), ambiguous_width) for c in data])
+            return sum(_EAW_MAP.get(east_asian_width(c), ambiguous_width) for c in data)
         else:
             return len(data)
 
@@ -318,7 +318,7 @@ def east_asian_len(data, encoding=None, ambiguous_width=1):
                 data = data.decode(encoding)
             except UnicodeError:
                 pass
-            return sum([_EAW_MAP.get(east_asian_width(c), ambiguous_width) for c in data])
+            return sum(_EAW_MAP.get(east_asian_width(c), ambiguous_width) for c in data)
         else:
             return len(data)
 

pandas/core/base.py

@@ -486,13 +486,13 @@ def _agg(arg, func):
 
             def is_any_series():
                 # return a boolean if we have *any* nested series
-                return any([isinstance(r, ABCSeries)
-                            for r in compat.itervalues(result)])
+                return any(isinstance(r, ABCSeries)
+                           for r in compat.itervalues(result))
 
             def is_any_frame():
                 # return a boolean if we have *any* nested series
-                return any([isinstance(r, ABCDataFrame)
-                            for r in compat.itervalues(result)])
+                return any(isinstance(r, ABCDataFrame)
+                           for r in compat.itervalues(result))
 
             if isinstance(result, list):
                 return concat(result, keys=keys, axis=1), True

pandas/core/config.py

@@ -800,7 +800,7 @@ def inner(x):
         from pandas.io.formats.printing import pprint_thing as pp
         if x not in legal_values:
 
-            if not any([c(x) for c in callables]):
+            if not any(c(x) for c in callables):
                 pp_values = pp("|".join(lmap(pp, legal_values)))
                 msg = "Value must be one of {pp_values}"
                 if len(callables):

pandas/core/frame.py

@@ -601,7 +601,7 @@ def _repr_fits_horizontal_(self, ignore_width=False):
 
         d.to_string(buf=buf)
         value = buf.getvalue()
-        repr_width = max([len(l) for l in value.split('\n')])
+        repr_width = max(len(l) for l in value.split('\n'))
 
         return repr_width < width
 
@@ -1798,7 +1798,7 @@ def info(self, verbose=None, buf=None, max_cols=None, memory_usage=None,
         def _verbose_repr():
             lines.append('Data columns (total %d columns):' %
                          len(self.columns))
-            space = max([len(pprint_thing(k)) for k in self.columns]) + 4
+            space = max(len(pprint_thing(k)) for k in self.columns) + 4
             counts = None
 
             tmpl = "%s%s"
@@ -5805,7 +5805,12 @@ def idxmin(self, axis=0, skipna=True):
             0 or 'index' for row-wise, 1 or 'columns' for column-wise
         skipna : boolean, default True
             Exclude NA/null values. If an entire row/column is NA, the result
-            will be NA
+            will be NA.
+
+        Raises
+        ------
+        ValueError
+            * If the row/column is empty
 
         Returns
         -------
@@ -5836,7 +5841,12 @@ def idxmax(self, axis=0, skipna=True):
             0 or 'index' for row-wise, 1 or 'columns' for column-wise
         skipna : boolean, default True
             Exclude NA/null values. If an entire row/column is NA, the result
-            will be first index.
+            will be NA.
+
+        Raises
+        ------
+        ValueError
+            * If the row/column is empty
 
         Returns
         -------
@@ -6414,7 +6424,7 @@ def convert(arr):
 
 
 def _get_names_from_index(data):
-    has_some_name = any([getattr(s, 'name', None) is not None for s in data])
+    has_some_name = any(getattr(s, 'name', None) is not None for s in data)
     if not has_some_name:
         return _default_index(len(data))
 

pandas/core/generic.py

@@ -1006,8 +1006,8 @@ def _set_axis_name(self, name, axis=0, inplace=False):
     # Comparisons
 
     def _indexed_same(self, other):
-        return all([self._get_axis(a).equals(other._get_axis(a))
-                    for a in self._AXIS_ORDERS])
+        return all(self._get_axis(a).equals(other._get_axis(a))
+                   for a in self._AXIS_ORDERS)
 
     def __neg__(self):
         values = _values_from_object(self)
@@ -2989,8 +2989,8 @@ def reindex(self, *args, **kwargs):
 
         # if all axes that are requested to reindex are equal, then only copy
         # if indicated must have index names equal here as well as values
-        if all([self._get_axis(axis).identical(ax)
-                for axis, ax in axes.items() if ax is not None]):
+        if all(self._get_axis(axis).identical(ax)
+               for axis, ax in axes.items() if ax is not None):
             if copy:
                 return self.copy()
             return self
@@ -5886,8 +5886,8 @@ def _where(self, cond, other=np.nan, inplace=False, axis=None, level=None,
 
                 # if we are NOT aligned, raise as we cannot where index
                 if (axis is None and
-                        not all([other._get_axis(i).equals(ax)
-                                 for i, ax in enumerate(self.axes)])):
+                        not all(other._get_axis(i).equals(ax)
+                                for i, ax in enumerate(self.axes))):
                     raise InvalidIndexError
 
             # slice me out of the other

pandas/core/groupby.py

@@ -3890,8 +3890,7 @@ def first_not_none(values):
             # values are not series or array-like but scalars
             else:
                 # only coerce dates if we find at least 1 datetime
-                coerce = True if any([isinstance(x, Timestamp)
-                                      for x in values]) else False
+                coerce = any(isinstance(x, Timestamp) for x in values)
                 # self._selection_name not passed through to Series as the
                 # result should not take the name of original selection
                 # of columns
@@ -4303,8 +4302,8 @@ def _reindex_output(self, result):
             return result
         elif len(groupings) == 1:
             return result
-        elif not any([isinstance(ping.grouper, (Categorical, CategoricalIndex))
-                      for ping in groupings]):
+        elif not any(isinstance(ping.grouper, (Categorical, CategoricalIndex))
+                     for ping in groupings):
             return result
 
         levels_list = [ping.group_index for ping in groupings]

pandas/core/indexes/base.py

@@ -907,7 +907,7 @@ def _extend_line(s, line, value, display_width, next_line_prefix):
 
         def best_len(values):
             if values:
-                return max([adj.len(x) for x in values])
+                return max(adj.len(x) for x in values)
             else:
                 return 0
 
@@ -4246,7 +4246,7 @@ def _trim_front(strings):
     Trims zeros and decimal points
     """
     trimmed = strings
-    while len(strings) > 0 and all([x[0] == ' ' for x in trimmed]):
+    while len(strings) > 0 and all(x[0] == ' ' for x in trimmed):
         trimmed = [x[1:] for x in trimmed]
     return trimmed