Merge branch 'master' of https://github.com/pandas-dev/pandas into overwrite_xl

Author: mirober

.github/workflows/ci.yml

@@ -154,9 +154,10 @@ jobs:
         PANDAS_DATA_MANAGER: array
       run: |
         source activate pandas-dev
+
         pytest pandas/tests/frame/methods
         pytest pandas/tests/frame/test_constructors.py
-        pytest pandas/tests/frame/constructors/
+        pytest pandas/tests/frame/test_*
         pytest pandas/tests/frame/test_reductions.py
         pytest pandas/tests/reductions/
         pytest pandas/tests/generic/test_generic.py
@@ -165,10 +166,30 @@ jobs:
         pytest pandas/tests/resample/
         pytest pandas/tests/reshape/merge
 
+        pytest pandas/tests/series/methods
+        pytest pandas/tests/series/test_*
+
         # indexing subset (temporary since other tests don't pass yet)
         pytest pandas/tests/frame/indexing/test_indexing.py::TestDataFrameIndexing::test_setitem_boolean
         pytest pandas/tests/frame/indexing/test_where.py
         pytest pandas/tests/frame/indexing/test_setitem.py::TestDataFrameSetItem::test_setitem_multi_index
         pytest pandas/tests/frame/indexing/test_setitem.py::TestDataFrameSetItem::test_setitem_listlike_indexer_duplicate_columns
         pytest pandas/tests/indexing/multiindex/test_setitem.py::TestMultiIndexSetItem::test_astype_assignment_with_dups
         pytest pandas/tests/indexing/multiindex/test_setitem.py::TestMultiIndexSetItem::test_frame_setitem_multi_column
+
+        pytest pandas/tests/api/
+        pytest pandas/tests/base/
+        pytest pandas/tests/computation/
+        pytest pandas/tests/config/
+        pytest pandas/tests/dtypes/
+        pytest pandas/tests/generic/
+        pytest pandas/tests/indexes/
+        pytest pandas/tests/libs/
+        pytest pandas/tests/plotting/
+        pytest pandas/tests/scalar/
+        pytest pandas/tests/strings/
+        pytest pandas/tests/tools/
+        pytest pandas/tests/tseries/
+        pytest pandas/tests/tslibs/
+        pytest pandas/tests/util/
+        pytest pandas/tests/window/

asv_bench/benchmarks/frame_methods.py

@@ -11,6 +11,7 @@
     date_range,
     isnull,
     period_range,
+    timedelta_range,
 )
 
 from .pandas_vb_common import tm
@@ -52,6 +53,7 @@ def setup(self):
         N = 10 ** 3
         self.df = DataFrame(np.random.randn(N * 10, N))
         self.idx = np.arange(4 * N, 7 * N)
+        self.idx_cols = np.random.randint(0, N, N)
         self.df2 = DataFrame(
             {
                 c: {
@@ -68,6 +70,9 @@ def time_reindex_axis0(self):
         self.df.reindex(self.idx)
 
     def time_reindex_axis1(self):
+        self.df.reindex(columns=self.idx_cols)
+
+    def time_reindex_axis1_missing(self):
         self.df.reindex(columns=self.idx)
 
     def time_reindex_both_axes(self):
@@ -351,15 +356,42 @@ def time_isnull_obj(self):
 
 class Fillna:
 
-    params = ([True, False], ["pad", "bfill"])
-    param_names = ["inplace", "method"]
-
-    def setup(self, inplace, method):
-        values = np.random.randn(10000, 100)
-        values[::2] = np.nan
-        self.df = DataFrame(values)
-
-    def time_frame_fillna(self, inplace, method):
+    params = (
+        [True, False],
+        ["pad", "bfill"],
+        [
+            "float64",
+            "float32",
+            "object",
+            "Int64",
+            "Float64",
+            "datetime64[ns]",
+            "datetime64[ns, tz]",
+            "timedelta64[ns]",
+        ],
+    )
+    param_names = ["inplace", "method", "dtype"]
+
+    def setup(self, inplace, method, dtype):
+        N, M = 10000, 100
+        if dtype in ("datetime64[ns]", "datetime64[ns, tz]", "timedelta64[ns]"):
+            data = {
+                "datetime64[ns]": date_range("2011-01-01", freq="H", periods=N),
+                "datetime64[ns, tz]": date_range(
+                    "2011-01-01", freq="H", periods=N, tz="Asia/Tokyo"
+                ),
+                "timedelta64[ns]": timedelta_range(start="1 day", periods=N, freq="1D"),
+            }
+            self.df = DataFrame({f"col_{i}": data[dtype] for i in range(M)})
+            self.df[::2] = None
+        else:
+            values = np.random.randn(N, M)
+            values[::2] = np.nan
+            if dtype == "Int64":
+                values = values.round()
+            self.df = DataFrame(values, dtype=dtype)
+
+    def time_frame_fillna(self, inplace, method, dtype):
         self.df.fillna(inplace=inplace, method=method)
 
 

asv_bench/benchmarks/io/json.py

@@ -6,6 +6,7 @@
     DataFrame,
     concat,
     date_range,
+    json_normalize,
     read_json,
     timedelta_range,
 )
@@ -77,6 +78,27 @@ def peakmem_read_json_lines_nrows(self, index):
         read_json(self.fname, orient="records", lines=True, nrows=15000)
 
 
+class NormalizeJSON(BaseIO):
+    fname = "__test__.json"
+    params = [
+        ["split", "columns", "index", "values", "records"],
+        ["df", "df_date_idx", "df_td_int_ts", "df_int_floats", "df_int_float_str"],
+    ]
+    param_names = ["orient", "frame"]
+
+    def setup(self, orient, frame):
+        data = {
+            "hello": ["thisisatest", 999898, "mixed types"],
+            "nest1": {"nest2": {"nest3": "nest3_value", "nest3_int": 3445}},
+            "nest1_list": {"nest2": ["blah", 32423, 546456.876, 92030234]},
+            "hello2": "string",
+        }
+        self.data = [data for i in range(10000)]
+
+    def time_normalize_json(self, orient, frame):
+        json_normalize(self.data)
+
+
 class ToJSON(BaseIO):
 
     fname = "__test__.json"

doc/source/whatsnew/v1.2.4.rst

@@ -15,7 +15,7 @@ including other versions of pandas.
 Fixed regressions
 ~~~~~~~~~~~~~~~~~
 
--
+- Fixed regression in :meth:`DataFrame.sum` when ``min_count`` greater than the :class:`DataFrame` shape was passed resulted in a ``ValueError`` (:issue:`39738`)
 -
 
 .. ---------------------------------------------------------------------------

doc/source/whatsnew/v1.3.0.rst

@@ -132,7 +132,7 @@ Other enhancements
 - Disallow :class:`DataFrame` indexer for ``iloc`` for :meth:`Series.__getitem__` and :meth:`DataFrame.__getitem__`, (:issue:`39004`)
 - :meth:`Series.apply` can now accept list-like or dictionary-like arguments that aren't lists or dictionaries, e.g. ``ser.apply(np.array(["sum", "mean"]))``, which was already the case for :meth:`DataFrame.apply` (:issue:`39140`)
 - :meth:`DataFrame.plot.scatter` can now accept a categorical column as the argument to ``c`` (:issue:`12380`, :issue:`31357`)
-- :meth:`.Styler.set_tooltips` allows on hover tooltips to be added to styled HTML dataframes (:issue:`35643`, :issue:`21266`, :issue:`39317`)
+- :meth:`.Styler.set_tooltips` allows on hover tooltips to be added to styled HTML dataframes (:issue:`35643`, :issue:`21266`, :issue:`39317`, :issue:`39708`)
 - :meth:`.Styler.set_tooltips_class` and :meth:`.Styler.set_table_styles` amended to optionally allow certain css-string input arguments (:issue:`39564`)
 - :meth:`.Styler.apply` now more consistently accepts ndarray function returns, i.e. in all cases for ``axis`` is ``0, 1 or None`` (:issue:`39359`)
 - :meth:`.Styler.apply` and :meth:`.Styler.applymap` now raise errors if wrong format CSS is passed on render (:issue:`39660`)
@@ -182,6 +182,46 @@ Preserve dtypes in  :meth:`~pandas.DataFrame.combine_first`
    combined.dtypes
 
 
+Try operating inplace when setting values with ``loc`` and ``iloc``
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+When setting an entire column using ``loc`` or ``iloc``, pandas will try to
+insert the values into the existing data rather than create an entirely new array.
+
+.. ipython:: python
+
+   df = pd.DataFrame(range(3), columns=["A"], dtype="float64")
+   values = df.values
+   new = np.array([5, 6, 7], dtype="int64")
+   df.loc[[0, 1, 2], "A"] = new
+
+In both the new and old behavior, the data in ``values`` is overwritten, but in
+the old behavior the dtype of ``df["A"]`` changed to ``int64``.
+
+*pandas 1.2.x*
+
+.. code-block:: ipython
+
+   In [1]: df.dtypes
+   Out[1]:
+   A    int64
+   dtype: object
+   In [2]: np.shares_memory(df["A"].values, new)
+   Out[2]: False
+   In [3]: np.shares_memory(df["A"].values, values)
+   Out[3]: False
+
+In pandas 1.3.0, ``df`` continues to share data with ``values``
+
+*pandas 1.3.0*
+
+.. ipython:: python
+
+   df.dtypes
+   np.shares_memory(df["A"], new)
+   np.shares_memory(df["A"], values)
+
+
 .. _whatsnew_130.notable_bug_fixes.setitem_with_bool_casting:
 
 Consistent Casting With Setting Into Boolean Series
@@ -324,6 +364,7 @@ Deprecations
 - Deprecated :meth:`core.window.ewm.ExponentialMovingWindow.vol` (:issue:`39220`)
 - Using ``.astype`` to convert between ``datetime64[ns]`` dtype and :class:`DatetimeTZDtype` is deprecated and will raise in a future version, use ``obj.tz_localize`` or ``obj.dt.tz_localize`` instead (:issue:`38622`)
 - Deprecated casting ``datetime.date`` objects to ``datetime64`` when used as ``fill_value`` in :meth:`DataFrame.unstack`, :meth:`DataFrame.shift`, :meth:`Series.shift`, and :meth:`DataFrame.reindex`, pass ``pd.Timestamp(dateobj)`` instead (:issue:`39767`)
+- Deprecated allowing partial failure in :meth:`Series.transform` and :meth:`DataFrame.transform` when ``func`` is list-like or dict-like; will raise if any function fails on a column in a future version (:issue:`40211`)
 
 .. ---------------------------------------------------------------------------
 
@@ -335,10 +376,12 @@ Performance improvements
 - Performance improvement in :meth:`IntervalIndex.isin` (:issue:`38353`)
 - Performance improvement in :meth:`Series.mean` for nullable data types (:issue:`34814`)
 - Performance improvement in :meth:`Series.isin` for nullable data types (:issue:`38340`)
+- Performance improvement in :meth:`DataFrame.fillna` with ``method="pad|backfill"`` for nullable floating and nullable integer dtypes (:issue:`39953`)
 - Performance improvement in :meth:`DataFrame.corr` for method=kendall (:issue:`28329`)
 - Performance improvement in :meth:`core.window.rolling.Rolling.corr` and :meth:`core.window.rolling.Rolling.cov` (:issue:`39388`)
 - Performance improvement in :meth:`core.window.rolling.RollingGroupby.corr`, :meth:`core.window.expanding.ExpandingGroupby.corr`, :meth:`core.window.expanding.ExpandingGroupby.corr` and :meth:`core.window.expanding.ExpandingGroupby.cov` (:issue:`39591`)
 - Performance improvement in :func:`unique` for object data type (:issue:`37615`)
+- Performance improvement in :func:`pd.json_normalize` for basic cases (including seperators) (:issue:`40035` :issue:`15621`)
 - Performance improvement in :class:`core.window.rolling.ExpandingGroupby` aggregation methods (:issue:`39664`)
 - Performance improvement in :class:`Styler` where render times are more than 50% reduced (:issue:`39972` :issue:`39952`)
 - Performance improvement in :meth:`core.window.ewm.ExponentialMovingWindow.mean` with ``times`` (:issue:`39784`)
@@ -540,6 +583,8 @@ Reshaping
 - Bug in :meth:`DataFrame.append` returning incorrect dtypes with combinations of ``ExtensionDtype`` dtypes (:issue:`39454`)
 - Bug in :meth:`DataFrame.append` returning incorrect dtypes with combinations of ``datetime64`` and ``timedelta64`` dtypes (:issue:`39574`)
 - Bug in :meth:`DataFrame.pivot_table` returning a ``MultiIndex`` for a single value when operating on and empty ``DataFrame`` (:issue:`13483`)
+- Allow :class:`Index` to be passed to the :func:`numpy.all` function (:issue:`40180`)
+-
 
 Sparse
 ^^^^^^

pandas/_libs/algos.pyx

@@ -597,10 +597,11 @@ def pad(ndarray[algos_t] old, ndarray[algos_t] new, limit=None):
 
 @cython.boundscheck(False)
 @cython.wraparound(False)
-def pad_inplace(algos_t[:] values, const uint8_t[:] mask, limit=None):
+def pad_inplace(algos_t[:] values, uint8_t[:] mask, limit=None):
     cdef:
         Py_ssize_t i, N
         algos_t val
+        uint8_t prev_mask
         int lim, fill_count = 0
 
     N = len(values)
@@ -612,15 +613,18 @@ def pad_inplace(algos_t[:] values, const uint8_t[:] mask, limit=None):
     lim = validate_limit(N, limit)
 
     val = values[0]
+    prev_mask = mask[0]
     for i in range(N):
         if mask[i]:
             if fill_count >= lim:
                 continue
             fill_count += 1
             values[i] = val
+            mask[i] = prev_mask
         else:
             fill_count = 0
             val = values[i]
+            prev_mask = mask[i]
 
 
 @cython.boundscheck(False)
@@ -739,10 +743,11 @@ def backfill(ndarray[algos_t] old, ndarray[algos_t] new, limit=None) -> ndarray:
 
 @cython.boundscheck(False)
 @cython.wraparound(False)
-def backfill_inplace(algos_t[:] values, const uint8_t[:] mask, limit=None):
+def backfill_inplace(algos_t[:] values, uint8_t[:] mask, limit=None):
     cdef:
         Py_ssize_t i, N
         algos_t val
+        uint8_t prev_mask
         int lim, fill_count = 0
 
     N = len(values)
@@ -754,15 +759,18 @@ def backfill_inplace(algos_t[:] values, const uint8_t[:] mask, limit=None):
     lim = validate_limit(N, limit)
 
     val = values[N - 1]
+    prev_mask = mask[N - 1]
     for i in range(N - 1, -1, -1):
         if mask[i]:
             if fill_count >= lim:
                 continue
             fill_count += 1
             values[i] = val
+            mask[i] = prev_mask
         else:
             fill_count = 0
             val = values[i]
+            prev_mask = mask[i]
 
 
 @cython.boundscheck(False)

pandas/_typing.py

@@ -58,6 +58,8 @@
     from pandas.core.internals import (
         ArrayManager,
         BlockManager,
+        SingleArrayManager,
+        SingleBlockManager,
     )
     from pandas.core.resample import Resampler
     from pandas.core.series import Series
@@ -184,3 +186,4 @@
 
 # internals
 Manager = Union["ArrayManager", "BlockManager"]
+SingleManager = Union["SingleArrayManager", "SingleBlockManager"]

pandas/compat/numpy/function.py

@@ -212,6 +212,7 @@ def validate_cum_func_with_skipna(skipna, args, kwargs, name):
 ALLANY_DEFAULTS["dtype"] = None
 ALLANY_DEFAULTS["out"] = None
 ALLANY_DEFAULTS["keepdims"] = False
+ALLANY_DEFAULTS["axis"] = None
 validate_all = CompatValidator(
     ALLANY_DEFAULTS, fname="all", method="both", max_fname_arg_count=1
 )

pandas/conftest.py

@@ -190,7 +190,7 @@ def add_imports(doctest_namespace):
 # ----------------------------------------------------------------
 # Common arguments
 # ----------------------------------------------------------------
-@pytest.fixture(params=[0, 1, "index", "columns"], ids=lambda x: f"axis {repr(x)}")
+@pytest.fixture(params=[0, 1, "index", "columns"], ids=lambda x: f"axis={repr(x)}")
 def axis(request):
     """
     Fixture for returning the axis numbers of a DataFrame.
@@ -1585,6 +1585,14 @@ def indexer_sl(request):
     return request.param
 
 
+@pytest.fixture(params=[tm.at, tm.loc])
+def indexer_al(request):
+    """
+    Parametrize over at.__setitem__, loc.__setitem__
+    """
+    return request.param
+
+
 @pytest.fixture
 def using_array_manager(request):
     """

pandas/core/apply.py

@@ -15,6 +15,7 @@
     Union,
     cast,
 )
+import warnings
 
 import numpy as np
 
@@ -267,6 +268,7 @@ def transform_dict_like(self, func):
         func = self.normalize_dictlike_arg("transform", obj, func)
 
         results: Dict[Hashable, FrameOrSeriesUnion] = {}
+        failed_names = []
         for name, how in func.items():
             colg = obj._gotitem(name, ndim=1)
             try:
@@ -277,10 +279,20 @@ def transform_dict_like(self, func):
                     "No transform functions were provided",
                 }:
                     raise err
-
+                else:
+                    failed_names.append(name)
         # combine results
         if not results:
             raise ValueError("Transform function failed")
+        if len(failed_names) > 0:
+            warnings.warn(
+                f"{failed_names} did not transform successfully. "
+                f"Allowing for partial failure is deprecated, this will raise "
+                f"a ValueError in a future version of pandas."
+                f"Drop these columns/ops to avoid this warning.",
+                FutureWarning,
+                stacklevel=4,
+            )
         return concat(results, axis=1)
 
     def transform_str_or_callable(self, func) -> FrameOrSeriesUnion: