Merge branch 'master' into GH35185

Author: jreback

ci/deps/azure-38.yaml

@@ -17,4 +17,4 @@ dependencies:
   - python-dateutil
   - nomkl
   - pytz
-  - tabulate==0.8.3
+  - tabulate==0.8.7

doc/source/getting_started/install.rst

@@ -283,7 +283,7 @@ pyreadstat                                   SPSS files (.sav) reading
 pyxlsb                    1.0.6              Reading for xlsb files
 qtpy                                         Clipboard I/O
 s3fs                      0.4.0              Amazon S3 access
-tabulate                  0.8.3              Printing in Markdown-friendly format (see `tabulate`_)
+tabulate                  0.8.7              Printing in Markdown-friendly format (see `tabulate`_)
 xarray                    0.12.3             pandas-like API for N-dimensional data
 xclip                                        Clipboard I/O on linux
 xlrd                      1.2.0              Excel reading

doc/source/reference/indexing.rst

@@ -290,6 +290,7 @@ MultiIndex properties
    MultiIndex.codes
    MultiIndex.nlevels
    MultiIndex.levshape
+   MultiIndex.dtypes
 
 MultiIndex components
 ~~~~~~~~~~~~~~~~~~~~~

doc/source/whatsnew/v1.2.0.rst

@@ -308,6 +308,7 @@ Other enhancements
 - Improved error reporting for subsetting columns of a :class:`.DataFrameGroupBy` with ``axis=1`` (:issue:`37725`)
 - Implement method ``cross`` for :meth:`DataFrame.merge` and :meth:`DataFrame.join` (:issue:`5401`)
 - When :func:`read_csv/sas/json` are called with ``chuncksize``/``iterator`` they can be used in a ``with`` statement as they return context-managers (:issue:`38225`)
+- Augmented the list of named colors available for styling Excel exports, enabling all of CSS4 colors (:issue:`38247`)
 
 .. ---------------------------------------------------------------------------
 

doc/source/whatsnew/v1.3.0.rst

@@ -19,8 +19,8 @@ Enhancements
 Other enhancements
 ^^^^^^^^^^^^^^^^^^
 
--
--
+- Added :meth:`MultiIndex.dtypes` (:issue:`37062`)
+- Improve error message when ``usecols`` and ``names`` do not match for :func:`read_csv` and ``engine="c"`` (:issue:`29042`)
 
 .. ---------------------------------------------------------------------------
 
@@ -37,7 +37,76 @@ These are bug fixes that might have notable behavior changes.
 
 Increased minimum versions for dependencies
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-
+Some minimum supported versions of dependencies were updated.
+If installed, we now require:
+
++-----------------+-----------------+----------+---------+
+| Package         | Minimum Version | Required | Changed |
++=================+=================+==========+=========+
+| numpy           | 1.16.5          |    X     |         |
++-----------------+-----------------+----------+---------+
+| pytz            | 2017.3          |    X     |         |
++-----------------+-----------------+----------+---------+
+| python-dateutil | 2.7.3           |    X     |         |
++-----------------+-----------------+----------+---------+
+| bottleneck      | 1.2.1           |          |         |
++-----------------+-----------------+----------+---------+
+| numexpr         | 2.6.8           |          |         |
++-----------------+-----------------+----------+---------+
+| pytest (dev)    | 5.0.1           |          |         |
++-----------------+-----------------+----------+---------+
+| mypy (dev)      | 0.782           |          |         |
++-----------------+-----------------+----------+---------+
+
+For `optional libraries <https://dev.pandas.io/docs/install.html#dependencies>`_ the general recommendation is to use the latest version.
+The following table lists the lowest version per library that is currently being tested throughout the development of pandas.
+Optional libraries below the lowest tested version may still work, but are not considered supported.
+
++-----------------+-----------------+---------+
+| Package         | Minimum Version | Changed |
++=================+=================+=========+
+| beautifulsoup4  | 4.6.0           |         |
++-----------------+-----------------+---------+
+| fastparquet     | 0.3.2           |         |
++-----------------+-----------------+---------+
+| fsspec          | 0.7.4           |         |
++-----------------+-----------------+---------+
+| gcsfs           | 0.6.0           |         |
++-----------------+-----------------+---------+
+| lxml            | 4.3.0           |         |
++-----------------+-----------------+---------+
+| matplotlib      | 2.2.3           |         |
++-----------------+-----------------+---------+
+| numba           | 0.46.0          |         |
++-----------------+-----------------+---------+
+| openpyxl        | 2.6.0           |         |
++-----------------+-----------------+---------+
+| pyarrow         | 0.15.0          |         |
++-----------------+-----------------+---------+
+| pymysql         | 0.7.11          |         |
++-----------------+-----------------+---------+
+| pytables        | 3.5.1           |         |
++-----------------+-----------------+---------+
+| s3fs            | 0.4.0           |         |
++-----------------+-----------------+---------+
+| scipy           | 1.2.0           |         |
++-----------------+-----------------+---------+
+| sqlalchemy      | 1.2.8           |         |
++-----------------+-----------------+---------+
+| tabulate        | 0.8.7           |    X    |
++-----------------+-----------------+---------+
+| xarray          | 0.12.0          |         |
++-----------------+-----------------+---------+
+| xlrd            | 1.2.0           |         |
++-----------------+-----------------+---------+
+| xlsxwriter      | 1.0.2           |         |
++-----------------+-----------------+---------+
+| xlwt            | 1.3.0           |         |
++-----------------+-----------------+---------+
+| pandas-gbq      | 0.12.0          |         |
++-----------------+-----------------+---------+
+
+See :ref:`install.dependencies` and :ref:`install.optional_dependencies` for more.
 
 .. _whatsnew_130.api.other:
 
@@ -86,7 +155,7 @@ Categorical
 
 Datetimelike
 ^^^^^^^^^^^^
-
+- Bug in :class:`DataFrame` and :class:`Series` constructors sometimes dropping nanoseconds from :class:`Timestamp` (resp. :class:`Timedelta`) ``data``, with ``dtype=datetime64[ns]`` (resp. ``timedelta64[ns]``) (:issue:`38032`)
 -
 -
 
@@ -127,7 +196,9 @@ Interval
 
 Indexing
 ^^^^^^^^
-
+- Bug in :meth:`CategoricalIndex.get_indexer` failing to raise ``InvalidIndexError`` when non-unique (:issue:`38372`)
+- Bug in inserting many new columns into a :class:`DataFrame` causing incorrect subsequent indexing behavior (:issue:`38380`)
+- Bug in :meth:`DataFrame.iloc.__setitem__` and :meth:`DataFrame.loc.__setitem__` with mixed dtypes when setting with a dictionary value (:issue:`38335`)
 -
 -
 
@@ -147,7 +218,8 @@ I/O
 ^^^
 
 - Allow custom error values for parse_dates argument of :func:`read_sql`, :func:`read_sql_query` and :func:`read_sql_table` (:issue:`GH35185`)
-
+- Bug in :func:`read_csv` not accepting ``usecols`` with different length than ``names`` for ``engine="python"`` (:issue:`16469`)
+- Bug in :func:`read_csv` raising ``TypeError`` when ``names`` and ``parse_dates`` is specified for ``engine="c"`` (:issue:`33699`)
 -
 
 Period

pandas/_libs/parsers.pyx

@@ -738,8 +738,8 @@ cdef class TextReader:
                 elif self.names is None and nuse < passed_count:
                     self.leading_cols = field_count - passed_count
                 elif passed_count != field_count:
-                    raise ValueError('Passed header names '
-                                     'mismatches usecols')
+                    raise ValueError('Number of passed names did not match number of '
+                                     'header fields in the file')
             # oh boy, #2442, #2981
             elif self.allow_leading_cols and passed_count < field_count:
                 self.leading_cols = field_count - passed_count

pandas/compat/_optional.py

@@ -24,7 +24,7 @@
     "scipy": "1.2.0",
     "sqlalchemy": "1.2.8",
     "tables": "3.5.1",
-    "tabulate": "0.8.3",
+    "tabulate": "0.8.7",
     "xarray": "0.12.3",
     "xlrd": "1.2.0",
     "xlwt": "1.3.0",

pandas/core/algorithms.py

@@ -50,7 +50,7 @@
 from pandas.core.dtypes.generic import (
     ABCDatetimeArray,
     ABCExtensionArray,
-    ABCIndexClass,
+    ABCIndex,
     ABCMultiIndex,
     ABCRangeIndex,
     ABCSeries,
@@ -63,6 +63,7 @@
 
 if TYPE_CHECKING:
     from pandas import Categorical, DataFrame, Index, Series
+    from pandas.core.arrays import DatetimeArray, TimedeltaArray
 
 _shared_docs: Dict[str, str] = {}
 
@@ -215,7 +216,7 @@ def _reconstruct_data(
         values = values.astype(dtype, copy=False)
 
         # we only support object dtypes bool Index
-        if isinstance(original, ABCIndexClass):
+        if isinstance(original, ABCIndex):
             values = values.astype(object, copy=False)
     elif dtype is not None:
         if is_datetime64_dtype(dtype):
@@ -437,9 +438,7 @@ def isin(comps: AnyArrayLike, values: AnyArrayLike) -> np.ndarray:
             f"to isin(), you passed a [{type(values).__name__}]"
         )
 
-    if not isinstance(
-        values, (ABCIndexClass, ABCSeries, ABCExtensionArray, np.ndarray)
-    ):
+    if not isinstance(values, (ABCIndex, ABCSeries, ABCExtensionArray, np.ndarray)):
         values = _ensure_arraylike(list(values))
     elif isinstance(values, ABCMultiIndex):
         # Avoid raising in extract_array
@@ -700,7 +699,7 @@ def factorize(
         and values.freq is not None
     ):
         codes, uniques = values.factorize(sort=sort)
-        if isinstance(original, ABCIndexClass):
+        if isinstance(original, ABCIndex):
             uniques = original._shallow_copy(uniques, name=None)
         elif isinstance(original, ABCSeries):
             from pandas import Index
@@ -739,8 +738,11 @@ def factorize(
     uniques = _reconstruct_data(uniques, dtype, original)
 
     # return original tenor
-    if isinstance(original, ABCIndexClass):
+    if isinstance(original, ABCIndex):
         if original.dtype.kind in ["m", "M"] and isinstance(uniques, np.ndarray):
+            original._data = cast(
+                "Union[DatetimeArray, TimedeltaArray]", original._data
+            )
             uniques = type(original._data)._simple_new(uniques, dtype=original.dtype)
         uniques = original._shallow_copy(uniques, name=None)
     elif isinstance(original, ABCSeries):

pandas/core/arrays/__init__.py

@@ -7,7 +7,7 @@
 from pandas.core.arrays.categorical import Categorical
 from pandas.core.arrays.datetimes import DatetimeArray
 from pandas.core.arrays.floating import FloatingArray
-from pandas.core.arrays.integer import IntegerArray, integer_array
+from pandas.core.arrays.integer import IntegerArray
 from pandas.core.arrays.interval import IntervalArray
 from pandas.core.arrays.masked import BaseMaskedArray
 from pandas.core.arrays.numpy_ import PandasArray, PandasDtype
@@ -26,7 +26,6 @@
     "DatetimeArray",
     "FloatingArray",
     "IntegerArray",
-    "integer_array",
     "IntervalArray",
     "PandasArray",
     "PandasDtype",

pandas/core/arrays/base.py

@@ -41,7 +41,7 @@
     pandas_dtype,
 )
 from pandas.core.dtypes.dtypes import ExtensionDtype
-from pandas.core.dtypes.generic import ABCDataFrame, ABCIndexClass, ABCSeries
+from pandas.core.dtypes.generic import ABCDataFrame, ABCIndex, ABCSeries
 from pandas.core.dtypes.missing import isna
 
 from pandas.core import ops
@@ -1361,7 +1361,7 @@ def convert_values(param):
                     ovalues = [param] * len(self)
                 return ovalues
 
-            if isinstance(other, (ABCSeries, ABCIndexClass, ABCDataFrame)):
+            if isinstance(other, (ABCSeries, ABCIndex, ABCDataFrame)):
                 # rely on pandas to unbox and dispatch to us
                 return NotImplemented