Merge branch 'master' of https://github.com/pandas-dev/pandas into overwrite_xl

Author: mirober

doc/source/reference/style.rst

@@ -35,7 +35,6 @@ Style application
    Styler.applymap
    Styler.where
    Styler.format
-   Styler.set_precision
    Styler.set_td_classes
    Styler.set_table_styles
    Styler.set_table_attributes
@@ -44,7 +43,6 @@ Style application
    Styler.set_caption
    Styler.set_properties
    Styler.set_uuid
-   Styler.set_na_rep
    Styler.clear
    Styler.pipe
 
@@ -53,9 +51,9 @@ Builtin styles
 .. autosummary::
    :toctree: api/
 
+   Styler.highlight_null
    Styler.highlight_max
    Styler.highlight_min
-   Styler.highlight_null
    Styler.background_gradient
    Styler.bar
 

doc/source/whatsnew/v1.3.0.rst

@@ -136,6 +136,7 @@ Other enhancements
 - :meth:`.Styler.set_tooltips_class` and :meth:`.Styler.set_table_styles` amended to optionally allow certain css-string input arguments (:issue:`39564`)
 - :meth:`.Styler.apply` now more consistently accepts ndarray function returns, i.e. in all cases for ``axis`` is ``0, 1 or None`` (:issue:`39359`)
 - :meth:`.Styler.apply` and :meth:`.Styler.applymap` now raise errors if wrong format CSS is passed on render (:issue:`39660`)
+- Builtin highlighting methods in :class:`Styler` have a more consistent signature and css customisability (:issue:`40242`)
 - :meth:`Series.loc.__getitem__` and :meth:`Series.loc.__setitem__` with :class:`MultiIndex` now raising helpful error message when indexer has too many dimensions (:issue:`35349`)
 - :meth:`pandas.read_stata` and :class:`StataReader` support reading data from compressed files.
 - Add support for parsing ``ISO 8601``-like timestamps with negative signs to :meth:`pandas.Timedelta` (:issue:`37172`)
@@ -364,6 +365,7 @@ Deprecations
 - Deprecated :meth:`core.window.ewm.ExponentialMovingWindow.vol` (:issue:`39220`)
 - Using ``.astype`` to convert between ``datetime64[ns]`` dtype and :class:`DatetimeTZDtype` is deprecated and will raise in a future version, use ``obj.tz_localize`` or ``obj.dt.tz_localize`` instead (:issue:`38622`)
 - Deprecated casting ``datetime.date`` objects to ``datetime64`` when used as ``fill_value`` in :meth:`DataFrame.unstack`, :meth:`DataFrame.shift`, :meth:`Series.shift`, and :meth:`DataFrame.reindex`, pass ``pd.Timestamp(dateobj)`` instead (:issue:`39767`)
+- Deprecated :meth:`.Styler.set_na_rep` and :meth:`.Styler.set_precision` in favour of :meth:`.Styler.format` with ``na_rep`` and ``precision`` as existing and new input arguments respectively (:issue:`40134`)
 - Deprecated allowing partial failure in :meth:`Series.transform` and :meth:`DataFrame.transform` when ``func`` is list-like or dict-like; will raise if any function fails on a column in a future version (:issue:`40211`)
 
 .. ---------------------------------------------------------------------------

pandas/_libs/internals.pyx

@@ -15,22 +15,24 @@ cimport numpy as cnp
 from numpy cimport (
     NPY_INT64,
     int64_t,
+    ndarray,
 )
 
 cnp.import_array()
 
 from pandas._libs.algos import ensure_int64
+from pandas._libs.util cimport is_integer_object
 
 
 @cython.final
 cdef class BlockPlacement:
     # __slots__ = '_as_slice', '_as_array', '_len'
     cdef:
         slice _as_slice
-        object _as_array
+        ndarray _as_array  # Note: this still allows `None`
         bint _has_slice, _has_array, _is_known_slice_like
 
-    def __init__(self, val):
+    def __cinit__(self, val):
         cdef:
             slice slc
 
@@ -39,7 +41,7 @@ cdef class BlockPlacement:
         self._has_slice = False
         self._has_array = False
 
-        if isinstance(val, int):
+        if is_integer_object(val):
             slc = slice(val, val + 1, 1)
             self._as_slice = slc
             self._has_slice = True
@@ -160,12 +162,12 @@ cdef class BlockPlacement:
             np.concatenate([self.as_array] + [o.as_array for o in others])
         )
 
-    cdef iadd(self, other):
+    cdef BlockPlacement iadd(self, other):
         cdef:
             slice s = self._ensure_has_slice()
             Py_ssize_t other_int, start, stop, step, l
 
-        if isinstance(other, int) and s is not None:
+        if is_integer_object(other) and s is not None:
             other_int = <Py_ssize_t>other
 
             if other_int == 0:
@@ -438,13 +440,13 @@ def get_blkno_placements(blknos, group: bool = True):
     """
     Parameters
     ----------
-    blknos : array of int64
+    blknos : np.ndarray[int64]
     group : bool, default True
 
     Returns
     -------
     iterator
-        yield (BlockPlacement, blkno)
+        yield (blkno, BlockPlacement)
     """
     blknos = ensure_int64(blknos)
 

pandas/core/array_algos/take.py

@@ -121,6 +121,43 @@ def _take_nd_ndarray(
     return out
 
 
+def take_1d(
+    arr: ArrayLike,
+    indexer: np.ndarray,
+    fill_value=None,
+    allow_fill: bool = True,
+) -> ArrayLike:
+    """
+    Specialized version for 1D arrays. Differences compared to take_nd:
+
+    - Assumes input (arr, indexer) has already been converted to numpy array / EA
+    - Only works for 1D arrays
+
+    To ensure the lowest possible overhead.
+
+    TODO(ArrayManager): mainly useful for ArrayManager, otherwise can potentially
+    be removed again if we don't end up with ArrayManager.
+    """
+    if not isinstance(arr, np.ndarray):
+        # ExtensionArray -> dispatch to their method
+        return arr.take(indexer, fill_value=fill_value, allow_fill=allow_fill)
+
+    indexer, dtype, fill_value, mask_info = _take_preprocess_indexer_and_fill_value(
+        arr, indexer, 0, None, fill_value, allow_fill
+    )
+
+    # at this point, it's guaranteed that dtype can hold both the arr values
+    # and the fill_value
+    out = np.empty(indexer.shape, dtype=dtype)
+
+    func = _get_take_nd_function(
+        arr.ndim, arr.dtype, out.dtype, axis=0, mask_info=mask_info
+    )
+    func(arr, indexer, out, fill_value)
+
+    return out
+
+
 def take_2d_multi(
     arr: np.ndarray, indexer: np.ndarray, fill_value=np.nan
 ) -> np.ndarray:

pandas/core/indexes/base.py

@@ -4534,7 +4534,6 @@ def __getitem__(self, key):
         # There's no custom logic to be implemented in __getslice__, so it's
         # not overloaded intentionally.
         getitem = self._data.__getitem__
-        promote = self._shallow_copy
 
         if is_scalar(key):
             key = com.cast_scalar_indexer(key, warn_float=True)
@@ -4543,7 +4542,9 @@ def __getitem__(self, key):
         if isinstance(key, slice):
             # This case is separated from the conditional above to avoid
             # pessimization of basic indexing.
-            return promote(getitem(key))
+            result = getitem(key)
+            # Going through simple_new for performance.
+            return type(self)._simple_new(result, name=self.name)
 
         if com.is_bool_indexer(key):
             key = np.asarray(key, dtype=bool)
@@ -4553,7 +4554,9 @@ def __getitem__(self, key):
             if np.ndim(result) > 1:
                 deprecate_ndim_indexing(result)
                 return result
-            return promote(result)
+            # NB: Using _constructor._simple_new would break if MultiIndex
+            #  didn't override __getitem__
+            return self._constructor._simple_new(result, name=self.name)
         else:
             return result
 

pandas/core/indexes/category.py

@@ -231,7 +231,7 @@ def _simple_new(cls, values: Categorical, name: Optional[Hashable] = None):
         result = object.__new__(cls)
 
         result._data = values
-        result.name = name
+        result._name = name
         result._cache = {}
 
         result._reset_identity()

pandas/core/indexes/multi.py

@@ -2076,15 +2076,16 @@ def __getitem__(self, key):
 
             return tuple(retval)
         else:
+            # in general cannot be sure whether the result will be sorted
+            sortorder = None
             if com.is_bool_indexer(key):
                 key = np.asarray(key, dtype=bool)
                 sortorder = self.sortorder
-            else:
-                # cannot be sure whether the result will be sorted
-                sortorder = None
-
-                if isinstance(key, Index):
-                    key = np.asarray(key)
+            elif isinstance(key, slice):
+                if key.step is None or key.step > 0:
+                    sortorder = self.sortorder
+            elif isinstance(key, Index):
+                key = np.asarray(key)
 
             new_codes = [level_codes[key] for level_codes in self.codes]
 

pandas/core/indexes/range.py

@@ -164,7 +164,7 @@ def _simple_new(cls, values: range, name: Hashable = None) -> RangeIndex:
         assert isinstance(values, range)
 
         result._range = values
-        result.name = name
+        result._name = name
         result._cache = {}
         result._reset_identity()
         return result

pandas/core/internals/array_manager.py

@@ -58,7 +58,7 @@
 
 import pandas.core.algorithms as algos
 from pandas.core.array_algos.quantile import quantile_compat
-from pandas.core.array_algos.take import take_nd
+from pandas.core.array_algos.take import take_1d
 from pandas.core.arrays import (
     DatetimeArray,
     ExtensionArray,
@@ -991,7 +991,7 @@ def _reindex_indexer(
         else:
             validate_indices(indexer, len(self._axes[0]))
             new_arrays = [
-                take_nd(
+                take_1d(
                     arr,
                     indexer,
                     allow_fill=True,
@@ -1073,7 +1073,7 @@ def unstack(self, unstacker, fill_value) -> ArrayManager:
         new_arrays = []
         for arr in self.arrays:
             for i in range(unstacker.full_shape[1]):
-                new_arr = take_nd(
+                new_arr = take_1d(
                     arr, new_indexer2D[:, i], allow_fill=True, fill_value=fill_value
                 )
                 new_arrays.append(new_arr)

pandas/core/internals/blocks.py

@@ -449,7 +449,7 @@ def _split_op_result(self, result) -> List[Block]:
             nbs = []
             for i, loc in enumerate(self.mgr_locs):
                 vals = result[i]
-                block = self.make_block(values=vals, placement=[loc])
+                block = self.make_block(values=vals, placement=loc)
                 nbs.append(block)
             return nbs
 

pandas/core/internals/managers.py

@@ -149,6 +149,9 @@ class BlockManager(DataManager):
     _blknos: np.ndarray
     _blklocs: np.ndarray
 
+    # Non-trivially faster than a property
+    ndim = 2  # overridden by SingleBlockManager
+
     def __init__(
         self,
         blocks: Sequence[Block],
@@ -173,6 +176,21 @@ def __init__(
         self._blknos = None
         self._blklocs = None
 
+    @classmethod
+    def _simple_new(cls, blocks: Tuple[Block, ...], axes: List[Index]):
+        """
+        Fastpath constructor; does NO validation.
+        """
+        obj = cls.__new__(cls)
+        obj.axes = axes
+        obj.blocks = blocks
+
+        # Populate known_consolidate, blknos, and blklocs lazily
+        obj._known_consolidated = False
+        obj._blknos = None
+        obj._blklocs = None
+        return obj
+
     @classmethod
     def from_blocks(cls, blocks: List[Block], axes: List[Index]):
         """
@@ -233,10 +251,6 @@ def __nonzero__(self) -> bool:
     def shape(self) -> Shape:
         return tuple(len(ax) for ax in self.axes)
 
-    @property
-    def ndim(self) -> int:
-        return len(self.axes)
-
     def _normalize_axis(self, axis):
         # switch axis to follow BlockManager logic
         if self.ndim == 2:
@@ -800,8 +814,7 @@ def get_slice(self, slobj: slice, axis: int = 0) -> BlockManager:
         new_axes = list(self.axes)
         new_axes[axis] = new_axes[axis][slobj]
 
-        bm = type(self)(new_blocks, new_axes, verify_integrity=False)
-        return bm
+        return type(self)._simple_new(tuple(new_blocks), new_axes)
 
     @property
     def nblocks(self) -> int:
@@ -1322,7 +1335,7 @@ def reindex_indexer(
 
     def _slice_take_blocks_ax0(
         self, slice_or_indexer, fill_value=lib.no_default, only_slice: bool = False
-    ):
+    ) -> List[Block]:
         """
         Slice/take blocks along axis=0.