Merge remote-tracking branch 'upstream/master' into hiding_data_columns_and_index

Author: attack68

.circleci/config.yml

@@ -652,7 +652,9 @@ class Rank:
     ]
 
     def setup(self, dtype):
-        self.df = DataFrame(np.random.randn(10000, 10), columns=range(10), dtype=dtype)
+        self.df = DataFrame(
+            np.random.randn(10000, 10).astype(dtype), columns=range(10), dtype=dtype
+        )
 
     def time_rank(self, dtype):
         self.df.rank()

asv_bench/benchmarks/frame_methods.py

@@ -12,41 +12,30 @@ if [[ "$(uname)" == "Linux" && -n "$LC_ALL" ]]; then
     echo
 fi
 
-MINICONDA_DIR="$HOME/miniconda3"
-
-
-if [ -d "$MINICONDA_DIR" ]; then
-    echo
-    echo "rm -rf "$MINICONDA_DIR""
-    rm -rf "$MINICONDA_DIR"
-fi
 
 echo "Install Miniconda"
-UNAME_OS=$(uname)
-if [[ "$UNAME_OS" == 'Linux' ]]; then
+DEFAULT_CONDA_URL="https://repo.continuum.io/miniconda/Miniconda3-latest"
+if [[ "$(uname -m)" == 'aarch64' ]]; then
+    CONDA_URL="https://github.com/conda-forge/miniforge/releases/download/4.10.1-4/Miniforge3-4.10.1-4-Linux-aarch64.sh"
+elif [[ "$(uname)" == 'Linux' ]]; then
     if [[ "$BITS32" == "yes" ]]; then
-        CONDA_OS="Linux-x86"
+        CONDA_URL="$DEFAULT_CONDA_URL-Linux-x86.sh"
     else
-        CONDA_OS="Linux-x86_64"
+        CONDA_URL="$DEFAULT_CONDA_URL-Linux-x86_64.sh"
     fi
-elif [[ "$UNAME_OS" == 'Darwin' ]]; then
-    CONDA_OS="MacOSX-x86_64"
+elif [[ "$(uname)" == 'Darwin' ]]; then
+    CONDA_URL="$DEFAULT_CONDA_URL-MacOSX-x86_64.sh"
 else
-  echo "OS $UNAME_OS not supported"
+  echo "OS $(uname) not supported"
   exit 1
 fi
-
-if [ "${TRAVIS_CPU_ARCH}" == "arm64" ]; then
-  CONDA_URL="https://github.com/conda-forge/miniforge/releases/download/4.8.5-1/Miniforge3-4.8.5-1-Linux-aarch64.sh"
-else
-  CONDA_URL="https://repo.continuum.io/miniconda/Miniconda3-latest-$CONDA_OS.sh"
-fi
+echo "Downloading $CONDA_URL"
 wget -q $CONDA_URL -O miniconda.sh
 chmod +x miniconda.sh
 
-# Installation path is required for ARM64 platform as miniforge script installs in path $HOME/miniforge3.
+MINICONDA_DIR="$HOME/miniconda3"
+rm -rf $MINICONDA_DIR
 ./miniconda.sh -b -p $MINICONDA_DIR
-
 export PATH=$MINICONDA_DIR/bin:$PATH
 
 echo

ci/deps/travis-37-arm64.yaml renamed to ci/deps/circle-37-arm64.yaml

@@ -443,7 +443,7 @@ In the new behavior, we get a new array, and retain an integer-dtyped ``5``:
 Consistent Casting With Setting Into Boolean Series
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
-Setting non-boolean values into a :class:`Series with ``dtype=bool`` consistently
+Setting non-boolean values into a :class:`Series` with ``dtype=bool`` consistently
 cast to ``dtype=object`` (:issue:`38709`)
 
 .. ipython:: python
@@ -643,6 +643,7 @@ Other API changes
 - Partially initialized :class:`CategoricalDtype` (i.e. those with ``categories=None`` objects will no longer compare as equal to fully initialized dtype objects.
 - Accessing ``_constructor_expanddim`` on a :class:`DataFrame` and ``_constructor_sliced`` on a :class:`Series` now raise an ``AttributeError``. Previously a ``NotImplementedError`` was raised (:issue:`38782`)
 - Added new ``engine`` and ``**engine_kwargs`` parameters to :meth:`DataFrame.to_sql` to support other future "SQL engines". Currently we still only use ``SQLAlchemy`` under the hood, but more engines are planned to be supported such as ``turbodbc`` (:issue:`36893`)
+- Removed redundant ``freq`` from :class:`PeriodIndex` string representation (:issue:`41653`)
 
 Build
 =====
@@ -694,11 +695,12 @@ Deprecations
 - Deprecated passing arguments as positional in :meth:`DataFrame.set_index` (other than ``"keys"``) (:issue:`41485`)
 - Deprecated passing arguments as positional (except for ``"levels"``) in :meth:`MultiIndex.set_levels` (:issue:`41485`)
 - Deprecated passing arguments as positional in :meth:`DataFrame.sort_index` and :meth:`Series.sort_index` (:issue:`41485`)
-- Deprecated passing arguments as positional in :meth:`DataFrame.drop_duplicates` (except for ``subset``), :meth:`Series.drop_duplicates`, :meth:`Index.drop_duplicates` and :meth:`MultiIndex.drop_duplicates`(:issue:`41485`)
+- Deprecated passing arguments as positional in :meth:`DataFrame.drop_duplicates` (except for ``subset``), :meth:`Series.drop_duplicates`, :meth:`Index.drop_duplicates` and :meth:`MultiIndex.drop_duplicates` (:issue:`41485`)
 - Deprecated passing arguments (apart from ``value``) as positional in :meth:`DataFrame.fillna` and :meth:`Series.fillna` (:issue:`41485`)
 - Deprecated passing arguments as positional in :meth:`DataFrame.reset_index` (other than ``"level"``) and :meth:`Series.reset_index` (:issue:`41485`)
 - Deprecated construction of :class:`Series` or :class:`DataFrame` with ``DatetimeTZDtype`` data and ``datetime64[ns]`` dtype.  Use ``Series(data).dt.tz_localize(None)`` instead (:issue:`41555`,:issue:`33401`)
 - Deprecated behavior of :class:`Series` construction with large-integer values and small-integer dtype silently overflowing; use ``Series(data).astype(dtype)`` instead (:issue:`41734`)
+- Deprecated behavior of :class:`DataFrame` construction with floating data and integer dtype casting even when lossy; in a future version this will remain floating, matching :class:`Series` behavior (:issue:`41770`)
 - Deprecated inference of ``timedelta64[ns]``, ``datetime64[ns]``, or ``DatetimeTZDtype`` dtypes in :class:`Series` construction when data containing strings is passed and no ``dtype`` is passed (:issue:`33558`)
 - In a future version, constructing :class:`Series` or :class:`DataFrame` with ``datetime64[ns]`` data and ``DatetimeTZDtype`` will treat the data as wall-times instead of as UTC times (matching DatetimeIndex behavior). To treat the data as UTC times, use ``pd.Series(data).dt.tz_localize("UTC").dt.tz_convert(dtype.tz)`` or ``pd.Series(data.view("int64"), dtype=dtype)`` (:issue:`33401`)
 - Deprecated passing arguments as positional in :meth:`DataFrame.set_axis` and :meth:`Series.set_axis` (other than ``"labels"``) (:issue:`41485`)
@@ -924,7 +926,7 @@ Indexing
 - Bug in :meth:`Series.__setitem__` raising ``ValueError`` when setting a :class:`Series` with a scalar indexer (:issue:`38303`)
 - Bug in :meth:`DataFrame.loc` dropping levels of :class:`MultiIndex` when :class:`DataFrame` used as input has only one row (:issue:`10521`)
 - Bug in :meth:`DataFrame.__getitem__` and :meth:`Series.__getitem__` always raising ``KeyError`` when slicing with existing strings an :class:`Index` with milliseconds (:issue:`33589`)
-- Bug in setting ``timedelta64`` or ``datetime64`` values into numeric :class:`Series` failing to cast to object dtype (:issue:`39086`, issue:`39619`)
+- Bug in setting ``timedelta64`` or ``datetime64`` values into numeric :class:`Series` failing to cast to object dtype (:issue:`39086`, :issue:`39619`)
 - Bug in setting :class:`Interval` values into a :class:`Series` or :class:`DataFrame` with mismatched :class:`IntervalDtype` incorrectly casting the new values to the existing dtype (:issue:`39120`)
 - Bug in setting ``datetime64`` values into a :class:`Series` with integer-dtype incorrect casting the datetime64 values to integers (:issue:`39266`)
 - Bug in setting ``np.datetime64("NaT")`` into a :class:`Series` with :class:`Datetime64TZDtype` incorrectly treating the timezone-naive value as timezone-aware (:issue:`39769`)
@@ -944,6 +946,7 @@ Indexing
 - Bug in :meth:`DataFrame.loc` returning :class:`MultiIndex` in wrong order if indexer has duplicates (:issue:`40978`)
 - Bug in :meth:`DataFrame.__setitem__` raising ``TypeError`` when using a str subclass as the column name with a :class:`DatetimeIndex` (:issue:`37366`)
 - Bug in :meth:`PeriodIndex.get_loc` failing to raise ``KeyError`` when given a :class:`Period` with a mismatched ``freq`` (:issue:`41670`)
+- Bug ``.loc.__getitem__`` with a :class:`UInt64Index` and negative-integer keys raising ``OverflowError`` instead of ``KeyError`` in some cases, wrapping around to positive integers in others (:issue:`41777`)
 
 Missing
 ^^^^^^^
@@ -1045,7 +1048,7 @@ Groupby/resample/rolling
 - Bug in :class:`core.window.ewm.ExponentialMovingWindowGroupby` where the times vector and values became out of sync for non-trivial groups (:issue:`40951`)
 - Bug in :meth:`Series.asfreq` and :meth:`DataFrame.asfreq` dropping rows when the index is not sorted (:issue:`39805`)
 - Bug in aggregation functions for :class:`DataFrame` not respecting ``numeric_only`` argument when ``level`` keyword was given (:issue:`40660`)
-- Bug in :meth:`SeriesGroupBy.aggregate` where using a user-defined function to aggregate a ``Series`` with an object-typed :class:`Index` causes an incorrect :class:`Index` shape (issue:`40014`)
+- Bug in :meth:`SeriesGroupBy.aggregate` where using a user-defined function to aggregate a ``Series`` with an object-typed :class:`Index` causes an incorrect :class:`Index` shape (:issue:`40014`)
 - Bug in :class:`core.window.RollingGroupby` where ``as_index=False`` argument in ``groupby`` was ignored (:issue:`39433`)
 - Bug in :meth:`.GroupBy.any` and :meth:`.GroupBy.all` raising ``ValueError`` when using with nullable type columns holding ``NA`` even with ``skipna=True`` (:issue:`40585`)
 - Bug in :meth:`GroupBy.cummin` and :meth:`GroupBy.cummax` incorrectly rounding integer values near the ``int64`` implementations bounds (:issue:`40767`)
@@ -1054,12 +1057,13 @@ Groupby/resample/rolling
 - Bug in :meth:`DataFrame.rolling` returning mean zero for all ``NaN`` window with ``min_periods=0`` if calculation is not numerical stable (:issue:`41053`)
 - Bug in :meth:`DataFrame.rolling` returning sum not zero for all ``NaN`` window with ``min_periods=0`` if calculation is not numerical stable (:issue:`41053`)
 - Bug in :meth:`SeriesGroupBy.agg` failing to retain ordered :class:`CategoricalDtype` on order-preserving aggregations (:issue:`41147`)
-- Bug in :meth:`DataFrameGroupBy.min` and :meth:`DataFrameGroupBy.max` with multiple object-dtype columns and ``numeric_only=False`` incorrectly raising ``ValueError`` (:issue:41111`)
+- Bug in :meth:`DataFrameGroupBy.min` and :meth:`DataFrameGroupBy.max` with multiple object-dtype columns and ``numeric_only=False`` incorrectly raising ``ValueError`` (:issue:`41111`)
 - Bug in :meth:`DataFrameGroupBy.rank` with the GroupBy object's ``axis=0`` and the ``rank`` method's keyword ``axis=1`` (:issue:`41320`)
 - Bug in :meth:`DataFrameGroupBy.__getitem__` with non-unique columns incorrectly returning a malformed :class:`SeriesGroupBy` instead of :class:`DataFrameGroupBy` (:issue:`41427`)
 - Bug in :meth:`DataFrameGroupBy.transform` with non-unique columns incorrectly raising ``AttributeError`` (:issue:`41427`)
 - Bug in :meth:`Resampler.apply` with non-unique columns incorrectly dropping duplicated columns (:issue:`41445`)
 - Bug in :meth:`SeriesGroupBy` aggregations incorrectly returning empty :class:`Series` instead of raising ``TypeError`` on aggregations that are invalid for its dtype, e.g. ``.prod`` with ``datetime64[ns]`` dtype (:issue:`41342`)
+- Bug in :class:`DataFrameGroupBy` aggregations incorrectly failing to drop columns with invalid dtypes for that aggregation when there are no valid columns (:issue:`41291`)
 - Bug in :meth:`DataFrame.rolling.__iter__` where ``on`` was not assigned to the index of the resulting objects (:issue:`40373`)
 - Bug in :meth:`DataFrameGroupBy.transform` and :meth:`DataFrameGroupBy.agg` with ``engine="numba"`` where ``*args`` were being cached with the user passed function (:issue:`41647`)
 

ci/setup_env.sh

@@ -516,7 +516,7 @@ def group_add(add_t[:, ::1] out,
                 val = values[i, j]
 
                 # not nan
-                if val == val:
+                if not checknull(val):
                     nobs[lab, j] += 1
 
                     if nobs[lab, j] == 1:

doc/source/whatsnew/v1.3.0.rst

@@ -106,7 +106,8 @@ cdef class IndexEngine:
 
         try:
             return self.mapping.get_item(val)
-        except (TypeError, ValueError):
+        except (TypeError, ValueError, OverflowError):
+            # GH#41775 OverflowError e.g. if we are uint64 and val is -1
             raise KeyError(val)
 
     cdef inline _get_loc_duplicates(self, object val):

pandas/_libs/groupby.pyx

@@ -11,7 +11,10 @@ from typing import (
 
 import numpy as np
 
-from pandas._typing import ArrayLike
+from pandas._typing import (
+    ArrayLike,
+    DtypeObj,
+)
 
 # placeholder until we can specify np.ndarray[object, ndim=2]
 ndarray_obj_2d = np.ndarray
@@ -52,8 +55,6 @@ def is_float_array(values: np.ndarray, skipna: bool = False): ...
 def is_integer_array(values: np.ndarray, skipna: bool = False): ...
 def is_bool_array(values: np.ndarray, skipna: bool = False): ...
 
-def fast_multiget(mapping: dict, keys: np.ndarray, default=np.nan) -> np.ndarray: ...
-
 def fast_unique_multiple_list_gen(gen: Generator, sort: bool = True) -> list: ...
 def fast_unique_multiple_list(lists: list, sort: bool = True) -> list: ...
 def fast_unique_multiple(arrays: list, sort: bool = True) -> list: ...
@@ -73,6 +74,7 @@ def maybe_convert_objects(
     convert_timedelta: bool = ...,
     convert_period: Literal[False] = ...,
     convert_to_nullable_integer: Literal[False] = ...,
+    dtype_if_all_nat: DtypeObj | None = ...,
 ) -> np.ndarray: ...
 
 @overload
@@ -85,6 +87,7 @@ def maybe_convert_objects(
     convert_timedelta: bool = ...,
     convert_period: bool = ...,
     convert_to_nullable_integer: Literal[True] = ...,
+    dtype_if_all_nat: DtypeObj | None = ...,
 ) -> ArrayLike: ...
 
 @overload
@@ -97,6 +100,7 @@ def maybe_convert_objects(
     convert_timedelta: bool = ...,
     convert_period: bool = ...,
     convert_to_nullable_integer: bool = ...,
+    dtype_if_all_nat: DtypeObj | None = ...,
 ) -> ArrayLike: ...
 
 @overload
@@ -109,6 +113,7 @@ def maybe_convert_objects(
     convert_timedelta: bool = ...,
     convert_period: Literal[True] = ...,
     convert_to_nullable_integer: bool = ...,
+    dtype_if_all_nat: DtypeObj | None = ...,
 ) -> ArrayLike: ...
 
 @overload
@@ -121,6 +126,7 @@ def maybe_convert_objects(
     convert_timedelta: bool = ...,
     convert_period: bool = ...,
     convert_to_nullable_integer: bool = ...,
+    dtype_if_all_nat: DtypeObj | None = ...,
 ) -> ArrayLike: ...
 
 @overload
@@ -184,11 +190,7 @@ def maybe_indices_to_slice(
     max_len: int,
 ) -> slice | np.ndarray: ...  # np.ndarray[np.uint8]
 
-def clean_index_list(obj: list) -> tuple[
-    list | np.ndarray,  # np.ndarray[object] | np.ndarray[np.int64]
-    bool,
-]: ...
-
+def is_all_arraylike(obj: list) -> bool: ...
 
 # -----------------------------------------------------------------
 # Functions which in reality take memoryviews