Merge branch 'master' of https://github.com/pandas-dev/pandas into cln-comp

Author: jbrockmendel

pandas/_libs/lib.pyx

@@ -125,7 +125,7 @@ def is_scalar(val: object) -> bool:
         - Interval
         - DateOffset
         - Fraction
-        - Number
+        - Number.
 
     Returns
     -------
@@ -867,9 +867,10 @@ def is_list_like(obj: object, allow_sets: bool = True):
 
     Parameters
     ----------
-    obj : The object to check
-    allow_sets : boolean, default True
-        If this parameter is False, sets will not be considered list-like
+    obj : object
+        The object to check.
+    allow_sets : bool, default True
+        If this parameter is False, sets will not be considered list-like.
 
         .. versionadded:: 0.24.0
 

pandas/core/arrays/interval.py

@@ -260,9 +260,9 @@ def _from_factorized(cls, values, original):
         Whether the intervals are closed on the left-side, right-side, both
         or neither.
     copy : bool, default False
-        copy the data
+        Copy the data.
     dtype : dtype or None, default None
-        If None, dtype will be inferred
+        If None, dtype will be inferred.
 
         .. versionadded:: 0.23.0
 
@@ -383,16 +383,16 @@ def from_arrays(cls, left, right, closed="right", copy=False, dtype=None):
     Parameters
     ----------
     data : array-like (1-dimensional)
-        Array of tuples
+        Array of tuples.
     closed : {'left', 'right', 'both', 'neither'}, default 'right'
         Whether the intervals are closed on the left-side, right-side, both
         or neither.
     copy : bool, default False
-        by-default copy the data, this is compat only and ignored
+        By-default copy the data, this is compat only and ignored.
     dtype : dtype or None, default None
-        If None, dtype will be inferred
+        If None, dtype will be inferred.
 
-        ..versionadded:: 0.23.0
+        .. versionadded:: 0.23.0
 
     Returns
     -------

pandas/core/dtypes/concat.py

@@ -185,13 +185,14 @@ def concat_categorical(to_concat, axis=0):
 
 def union_categoricals(to_union, sort_categories=False, ignore_order=False):
     """
-    Combine list-like of Categorical-like, unioning categories. All
-    categories must have the same dtype.
+    Combine list-like of Categorical-like, unioning categories.
+
+    All categories must have the same dtype.
 
     Parameters
     ----------
-    to_union : list-like of Categorical, CategoricalIndex,
-               or Series with dtype='category'
+    to_union : list-like
+        Categorical, CategoricalIndex, or Series with dtype='category'.
     sort_categories : bool, default False
         If true, resulting categories will be lexsorted, otherwise
         they will be ordered as they appear in the data.
@@ -201,7 +202,7 @@ def union_categoricals(to_union, sort_categories=False, ignore_order=False):
 
     Returns
     -------
-    result : Categorical
+    Categorical
 
     Raises
     ------

pandas/core/groupby/generic.py

@@ -655,16 +655,17 @@ def value_counts(
         rep = partial(np.repeat, repeats=np.add.reduceat(inc, idx))
 
         # multi-index components
-        labels = list(map(rep, self.grouper.recons_labels)) + [llab(lab, inc)]
+        codes = self.grouper.recons_codes
+        codes = [rep(level_codes) for level_codes in codes] + [llab(lab, inc)]
         levels = [ping.group_index for ping in self.grouper.groupings] + [lev]
         names = self.grouper.names + [self._selection_name]
 
         if dropna:
-            mask = labels[-1] != -1
+            mask = codes[-1] != -1
             if mask.all():
                 dropna = False
             else:
-                out, labels = out[mask], [label[mask] for label in labels]
+                out, codes = out[mask], [level_codes[mask] for level_codes in codes]
 
         if normalize:
             out = out.astype("float")
@@ -680,11 +681,11 @@ def value_counts(
         if sort and bins is None:
             cat = ids[inc][mask] if dropna else ids[inc]
             sorter = np.lexsort((out if ascending else -out, cat))
-            out, labels[-1] = out[sorter], labels[-1][sorter]
+            out, codes[-1] = out[sorter], codes[-1][sorter]
 
         if bins is None:
             mi = MultiIndex(
-                levels=levels, codes=labels, names=names, verify_integrity=False
+                levels=levels, codes=codes, names=names, verify_integrity=False
             )
 
             if is_integer_dtype(out):
@@ -694,14 +695,14 @@ def value_counts(
         # for compat. with libgroupby.value_counts need to ensure every
         # bin is present at every index level, null filled with zeros
         diff = np.zeros(len(out), dtype="bool")
-        for lab in labels[:-1]:
-            diff |= np.r_[True, lab[1:] != lab[:-1]]
+        for level_codes in codes[:-1]:
+            diff |= np.r_[True, level_codes[1:] != level_codes[:-1]]
 
         ncat, nbin = diff.sum(), len(levels[-1])
 
         left = [np.repeat(np.arange(ncat), nbin), np.tile(np.arange(nbin), ncat)]
 
-        right = [diff.cumsum() - 1, labels[-1]]
+        right = [diff.cumsum() - 1, codes[-1]]
 
         _, idx = _get_join_indexers(left, right, sort=False, how="left")
         out = np.where(idx != -1, out[idx], 0)
@@ -711,7 +712,10 @@ def value_counts(
             out, left[-1] = out[sorter], left[-1][sorter]
 
         # build the multi-index w/ full levels
-        codes = list(map(lambda lab: np.repeat(lab[diff], nbin), labels[:-1]))
+        def build_codes(lev_codes: np.ndarray) -> np.ndarray:
+            return np.repeat(lev_codes[diff], nbin)
+
+        codes = [build_codes(lev_codes) for lev_codes in codes[:-1]]
         codes.append(left[-1])
 
         mi = MultiIndex(levels=levels, codes=codes, names=names, verify_integrity=False)
@@ -758,7 +762,7 @@ def pct_change(self, periods=1, fill_method="pad", limit=None, freq=None):
                 )
             )
         filled = getattr(self, fill_method)(limit=limit)
-        fill_grp = filled.groupby(self.grouper.labels)
+        fill_grp = filled.groupby(self.grouper.codes)
         shifted = fill_grp.shift(periods=periods, freq=freq)
 
         return (filled / shifted) - 1

pandas/core/groupby/groupby.py

@@ -899,10 +899,21 @@ def _python_agg_general(self, func, *args, **kwargs):
         output = {}
         for name, obj in self._iterate_slices():
             try:
-                result, counts = self.grouper.agg_series(obj, f)
+                # if this function is invalid for this dtype, we will ignore it.
+                func(obj[:0])
             except TypeError:
                 continue
-            else:
+            except AssertionError:
+                raise
+            except Exception:
+                # Our function depends on having a non-empty argument
+                #  See test_groupby_agg_err_catching
+                pass
+
+            result, counts = self.grouper.agg_series(obj, f)
+            if result is not None:
+                # TODO: only 3 test cases get None here, do something
+                #  in those cases
                 output[name] = self._try_cast(result, obj, numeric_only=True)
 
         if len(output) == 0:
@@ -2338,7 +2349,7 @@ def pct_change(self, periods=1, fill_method="pad", limit=None, freq=None, axis=0
                 )
             )
         filled = getattr(self, fill_method)(limit=limit)
-        fill_grp = filled.groupby(self.grouper.labels)
+        fill_grp = filled.groupby(self.grouper.codes)
         shifted = fill_grp.shift(periods=periods, freq=freq)
         return (filled / shifted) - 1
 

pandas/core/groupby/grouper.py

@@ -3,7 +3,7 @@
 split-apply-combine paradigm.
 """
 
-from typing import Tuple
+from typing import Optional, Tuple
 import warnings
 
 import numpy as np
@@ -21,6 +21,7 @@
 )
 from pandas.core.dtypes.generic import ABCSeries
 
+from pandas._typing import FrameOrSeries
 import pandas.core.algorithms as algorithms
 from pandas.core.arrays import Categorical, ExtensionArray
 import pandas.core.common as com
@@ -228,10 +229,10 @@ class Grouping:
     ----------
     index : Index
     grouper :
-    obj :
+    obj Union[DataFrame, Series]:
     name :
     level :
-    observed : boolean, default False
+    observed : bool, default False
         If we are a Categorical, use the observed values
     in_axis : if the Grouping is a column in self.obj and hence among
         Groupby.exclusions list
@@ -240,25 +241,22 @@ class Grouping:
     -------
     **Attributes**:
       * indices : dict of {group -> index_list}
-      * labels : ndarray, group labels
-      * ids : mapping of label -> group
-      * counts : array of group counts
+      * codes : ndarray, group codes
       * group_index : unique groups
       * groups : dict of {group -> label_list}
     """
 
     def __init__(
         self,
-        index,
+        index: Index,
         grouper=None,
-        obj=None,
+        obj: Optional[FrameOrSeries] = None,
         name=None,
         level=None,
-        sort=True,
-        observed=False,
-        in_axis=False,
+        sort: bool = True,
+        observed: bool = False,
+        in_axis: bool = False,
     ):
-
         self.name = name
         self.level = level
         self.grouper = _convert_grouper(index, grouper)
@@ -290,12 +288,12 @@ def __init__(
             if self.name is None:
                 self.name = index.names[level]
 
-            self.grouper, self._labels, self._group_index = index._get_grouper_for_level(  # noqa: E501
+            self.grouper, self._codes, self._group_index = index._get_grouper_for_level(  # noqa: E501
                 self.grouper, level
             )
 
         # a passed Grouper like, directly get the grouper in the same way
-        # as single grouper groupby, use the group_info to get labels
+        # as single grouper groupby, use the group_info to get codes
         elif isinstance(self.grouper, Grouper):
             # get the new grouper; we already have disambiguated
             # what key/level refer to exactly, don't need to
@@ -308,7 +306,7 @@ def __init__(
             self.grouper = grouper._get_grouper()
 
         else:
-            if self.grouper is None and self.name is not None:
+            if self.grouper is None and self.name is not None and self.obj is not None:
                 self.grouper = self.obj[self.name]
 
             elif isinstance(self.grouper, (list, tuple)):
@@ -324,7 +322,7 @@ def __init__(
 
                 # we make a CategoricalIndex out of the cat grouper
                 # preserving the categories / ordered attributes
-                self._labels = self.grouper.codes
+                self._codes = self.grouper.codes
                 if observed:
                     codes = algorithms.unique1d(self.grouper.codes)
                     codes = codes[codes != -1]
@@ -380,11 +378,11 @@ def __repr__(self):
     def __iter__(self):
         return iter(self.indices)
 
-    _labels = None
-    _group_index = None
+    _codes = None  # type: np.ndarray
+    _group_index = None  # type: Index
 
     @property
-    def ngroups(self):
+    def ngroups(self) -> int:
         return len(self.group_index)
 
     @cache_readonly
@@ -397,38 +395,38 @@ def indices(self):
         return values._reverse_indexer()
 
     @property
-    def labels(self):
-        if self._labels is None:
-            self._make_labels()
-        return self._labels
+    def codes(self) -> np.ndarray:
+        if self._codes is None:
+            self._make_codes()
+        return self._codes
 
     @cache_readonly
-    def result_index(self):
+    def result_index(self) -> Index:
         if self.all_grouper is not None:
             return recode_from_groupby(self.all_grouper, self.sort, self.group_index)
         return self.group_index
 
     @property
-    def group_index(self):
+    def group_index(self) -> Index:
         if self._group_index is None:
-            self._make_labels()
+            self._make_codes()
         return self._group_index
 
-    def _make_labels(self):
-        if self._labels is None or self._group_index is None:
+    def _make_codes(self) -> None:
+        if self._codes is None or self._group_index is None:
             # we have a list of groupers
             if isinstance(self.grouper, BaseGrouper):
-                labels = self.grouper.label_info
+                codes = self.grouper.codes_info
                 uniques = self.grouper.result_index
             else:
-                labels, uniques = algorithms.factorize(self.grouper, sort=self.sort)
+                codes, uniques = algorithms.factorize(self.grouper, sort=self.sort)
                 uniques = Index(uniques, name=self.name)
-            self._labels = labels
+            self._codes = codes
             self._group_index = uniques
 
     @cache_readonly
-    def groups(self):
-        return self.index.groupby(Categorical.from_codes(self.labels, self.group_index))
+    def groups(self) -> dict:
+        return self.index.groupby(Categorical.from_codes(self.codes, self.group_index))
 
 
 def _get_grouper(
@@ -678,7 +676,7 @@ def _is_label_like(val):
     return isinstance(val, (str, tuple)) or (val is not None and is_scalar(val))
 
 
-def _convert_grouper(axis, grouper):
+def _convert_grouper(axis: Index, grouper):
     if isinstance(grouper, dict):
         return grouper.get
     elif isinstance(grouper, Series):