Merge remote-tracking branch 'upstream/main' into finalize

Author: yuanx749

.github/workflows/docbuild-and-upload.yml

@@ -81,7 +81,7 @@ jobs:
       if: github.event_name == 'push' && github.ref == 'refs/heads/main'
 
     - name: Upload prod docs
-      run: rsync -az --delete doc/build/html/ docs@${{ secrets.server_ip }}:/usr/share/nginx/pandas/pandas-docs/version/${GITHUB_REF_NAME}
+      run: rsync -az --delete doc/build/html/ docs@${{ secrets.server_ip }}:/usr/share/nginx/pandas/pandas-docs/version/${GITHUB_REF_NAME:1}
       if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/')
 
     - name: Move docs into site directory

.pre-commit-config.yaml

@@ -258,10 +258,10 @@ repos:
             |/_testing/
     -   id: autotyping
         name: autotyping
-        entry: python -m libcst.tool codemod autotyping.AutotypeCommand --none-return --scalar-return --annotate-magics --annotate-imprecise-magics
+        entry: python -m libcst.tool codemod autotyping.AutotypeCommand --none-return --scalar-return --annotate-magics --annotate-imprecise-magics --bool-param
         types_or: [python, pyi]
         files: ^pandas
-        exclude: ^(pandas/tests|pandas/io/clipboard)
+        exclude: ^(pandas/tests|pandas/_version.py|pandas/io/clipboard)
         language: python
         additional_dependencies:
         - autotyping==22.9.0

asv_bench/benchmarks/index_object.py

@@ -19,28 +19,38 @@
 class SetOperations:
 
     params = (
-        ["datetime", "date_string", "int", "strings"],
+        ["monotonic", "non_monotonic"],
+        ["datetime", "date_string", "int", "strings", "ea_int"],
         ["intersection", "union", "symmetric_difference"],
     )
-    param_names = ["dtype", "method"]
+    param_names = ["index_structure", "dtype", "method"]
 
-    def setup(self, dtype, method):
+    def setup(self, index_structure, dtype, method):
         N = 10**5
         dates_left = date_range("1/1/2000", periods=N, freq="T")
         fmt = "%Y-%m-%d %H:%M:%S"
         date_str_left = Index(dates_left.strftime(fmt))
         int_left = Index(np.arange(N))
+        ea_int_left = Index(np.arange(N), dtype="Int64")
         str_left = tm.makeStringIndex(N)
+
         data = {
-            "datetime": {"left": dates_left, "right": dates_left[:-1]},
-            "date_string": {"left": date_str_left, "right": date_str_left[:-1]},
-            "int": {"left": int_left, "right": int_left[:-1]},
-            "strings": {"left": str_left, "right": str_left[:-1]},
+            "datetime": dates_left,
+            "date_string": date_str_left,
+            "int": int_left,
+            "strings": str_left,
+            "ea_int": ea_int_left,
         }
+
+        if index_structure == "non_monotonic":
+            data = {k: mi[::-1] for k, mi in data.items()}
+
+        data = {k: {"left": idx, "right": idx[:-1]} for k, idx in data.items()}
+
         self.left = data[dtype]["left"]
         self.right = data[dtype]["right"]
 
-    def time_operation(self, dtype, method):
+    def time_operation(self, index_structure, dtype, method):
         getattr(self.left, method)(self.right)
 
 

asv_bench/benchmarks/join_merge.py

@@ -6,6 +6,7 @@
     DataFrame,
     MultiIndex,
     Series,
+    array,
     concat,
     date_range,
     merge,
@@ -411,6 +412,42 @@ def time_multiby(self, direction, tolerance):
         )
 
 
+class MergeMultiIndex:
+    params = [
+        [
+            ("int64", "int64"),
+            ("datetime64[ns]", "int64"),
+            ("Int64", "Int64"),
+        ],
+        ["left", "right", "inner", "outer"],
+    ]
+    param_names = ["dtypes", "how"]
+
+    def setup(self, dtypes, how):
+        n = 100_000
+        offset = 50_000
+        mi1 = MultiIndex.from_arrays(
+            [
+                array(np.arange(n), dtype=dtypes[0]),
+                array(np.arange(n), dtype=dtypes[1]),
+            ]
+        )
+        mi2 = MultiIndex.from_arrays(
+            [
+                array(np.arange(offset, n + offset), dtype=dtypes[0]),
+                array(np.arange(offset, n + offset), dtype=dtypes[1]),
+            ]
+        )
+        self.df1 = DataFrame({"col1": 1}, index=mi1)
+        self.df2 = DataFrame({"col2": 2}, index=mi2)
+
+    def time_merge_sorted_multiindex(self, dtypes, how):
+        # copy to avoid MultiIndex._values caching
+        df1 = self.df1.copy()
+        df2 = self.df2.copy()
+        merge(df1, df2, how=how, left_index=True, right_index=True)
+
+
 class Align:
     def setup(self):
         size = 5 * 10**5

asv_bench/benchmarks/multiindex_object.py

@@ -237,7 +237,7 @@ class SetOperations:
 
     params = [
         ("monotonic", "non_monotonic"),
-        ("datetime", "int", "string"),
+        ("datetime", "int", "string", "ea_int"),
         ("intersection", "union", "symmetric_difference"),
     ]
     param_names = ["index_structure", "dtype", "method"]
@@ -255,10 +255,14 @@ def setup(self, index_structure, dtype, method):
         level2 = tm.makeStringIndex(N // 1000).values
         str_left = MultiIndex.from_product([level1, level2])
 
+        level2 = range(N // 1000)
+        ea_int_left = MultiIndex.from_product([level1, Series(level2, dtype="Int64")])
+
         data = {
             "datetime": dates_left,
             "int": int_left,
             "string": str_left,
+            "ea_int": ea_int_left,
         }
 
         if index_structure == "non_monotonic":
@@ -299,4 +303,36 @@ def time_unique_dups(self, dtype_val):
         self.midx_dups.unique()
 
 
+class Isin:
+    params = [
+        ("string", "int", "datetime"),
+    ]
+    param_names = ["dtype"]
+
+    def setup(self, dtype):
+        N = 10**5
+        level1 = range(1000)
+
+        level2 = date_range(start="1/1/2000", periods=N // 1000)
+        dates_midx = MultiIndex.from_product([level1, level2])
+
+        level2 = range(N // 1000)
+        int_midx = MultiIndex.from_product([level1, level2])
+
+        level2 = tm.makeStringIndex(N // 1000).values
+        str_midx = MultiIndex.from_product([level1, level2])
+
+        data = {
+            "datetime": dates_midx,
+            "int": int_midx,
+            "string": str_midx,
+        }
+
+        self.midx = data[dtype]
+        self.values = self.midx[:100]
+
+    def time_isin(self, dtype):
+        self.midx.isin(self.values)
+
+
 from .pandas_vb_common import setup  # noqa: F401 isort:skip

doc/_templates/sidebar-nav-bs.html

@@ -1,5 +1,5 @@
 <nav class="bd-links" id="bd-docs-nav" aria-label="Main navigation">
-  <div class="bd-toc-item active">
+  <div class="bd-toc-item navbar-nav">
     {% if pagename.startswith("reference") %}
     {{ generate_toctree_html("sidebar", maxdepth=4, collapse=True, includehidden=True, titles_only=True) }}
     {% else %}

doc/source/whatsnew/index.rst

@@ -24,6 +24,7 @@ Version 1.5
 .. toctree::
    :maxdepth: 2
 
+   v1.5.1
    v1.5.0
 
 Version 1.4

doc/source/whatsnew/v1.5.0.rst

@@ -1,7 +1,7 @@
 .. _whatsnew_150:
 
-What's new in 1.5.0 (??)
-------------------------
+What's new in 1.5.0 (September 19, 2022)
+----------------------------------------
 
 These are the changes in pandas 1.5.0. See :ref:`release` for a full changelog
 including other versions of pandas.
@@ -1214,12 +1214,10 @@ Sparse
 ^^^^^^
 - Bug in :meth:`Series.where` and :meth:`DataFrame.where` with ``SparseDtype`` failing to retain the array's ``fill_value`` (:issue:`45691`)
 - Bug in :meth:`SparseArray.unique` fails to keep original elements order (:issue:`47809`)
--
 
 ExtensionArray
 ^^^^^^^^^^^^^^
 - Bug in :meth:`IntegerArray.searchsorted` and :meth:`FloatingArray.searchsorted` returning inconsistent results when acting on ``np.nan`` (:issue:`45255`)
--
 
 Styler
 ^^^^^^
@@ -1234,18 +1232,18 @@ Metadata
 ^^^^^^^^
 - Fixed metadata propagation in :meth:`DataFrame.melt` (:issue:`28283`)
 - Fixed metadata propagation in :meth:`DataFrame.explode` (:issue:`28283`)
--
 
 Other
 ^^^^^
 
 .. ***DO NOT USE THIS SECTION***
 
 - Bug in :func:`.assert_index_equal` with ``names=True`` and ``check_order=False`` not checking names (:issue:`47328`)
--
 
 .. ---------------------------------------------------------------------------
 .. _whatsnew_150.contributors:
 
 Contributors
 ~~~~~~~~~~~~
+
+.. contributors:: v1.4.4..v1.5.0|HEAD

doc/source/whatsnew/v1.5.1.rst

@@ -0,0 +1,43 @@
+.. _whatsnew_151:
+
+What's new in 1.5.1 (October ??, 2022)
+--------------------------------------
+
+These are the changes in pandas 1.5.1. See :ref:`release` for a full changelog
+including other versions of pandas.
+
+{{ header }}
+
+.. ---------------------------------------------------------------------------
+
+.. _whatsnew_151.regressions:
+
+Fixed regressions
+~~~~~~~~~~~~~~~~~
+-
+-
+
+.. ---------------------------------------------------------------------------
+
+.. _whatsnew_151.bug_fixes:
+
+Bug fixes
+~~~~~~~~~
+-
+-
+
+.. ---------------------------------------------------------------------------
+
+.. _whatsnew_151.other:
+
+Other
+~~~~~
+-
+-
+
+.. ---------------------------------------------------------------------------
+
+.. _whatsnew_151.contributors:
+
+Contributors
+~~~~~~~~~~~~

doc/source/whatsnew/v1.6.0.rst

@@ -28,10 +28,12 @@ enhancement2
 
 Other enhancements
 ^^^^^^^^^^^^^^^^^^
+- :func:`read_sas` now supports using ``encoding='infer'`` to correctly read and use the encoding specified by the sas file. (:issue:`48048`)
 - :meth:`.DataFrameGroupBy.quantile` and :meth:`.SeriesGroupBy.quantile` now preserve nullable dtypes instead of casting to numpy dtypes (:issue:`37493`)
 - :meth:`Series.add_suffix`, :meth:`DataFrame.add_suffix`, :meth:`Series.add_prefix` and :meth:`DataFrame.add_prefix` support an ``axis`` argument. If ``axis`` is set, the default behaviour of which axis to consider can be overwritten (:issue:`47819`)
 - :func:`assert_frame_equal` now shows the first element where the DataFrames differ, analogously to ``pytest``'s output (:issue:`47910`)
 - Added ``index`` parameter to :meth:`DataFrame.to_dict` (:issue:`46398`)
+- Added metadata propagation for binary operators on :class:`DataFrame` (:issue:`28283`)
 -
 
 .. ---------------------------------------------------------------------------
@@ -44,8 +46,33 @@ These are bug fixes that might have notable behavior changes.
 
 .. _whatsnew_160.notable_bug_fixes.notable_bug_fix1:
 
-notable_bug_fix1
-^^^^^^^^^^^^^^^^
+:meth:`.GroupBy.cumsum` and :meth:`.GroupBy.cumprod` overflow instead of lossy casting to float
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+In previous versions we cast to float when applying ``cumsum`` and ``cumprod`` which
+lead to incorrect results even if the result could be hold by ``int64`` dtype.
+Additionally, the aggregation overflows consistent with numpy and the regular
+:meth:`DataFrame.cumprod` and :meth:`DataFrame.cumsum` methods when the limit of
+``int64`` is reached (:issue:`37493`).
+
+*Old Behavior*
+
+.. code-block:: ipython
+
+    In [1]: df = pd.DataFrame({"key": ["b"] * 7, "value": 625})
+    In [2]: df.groupby("key")["value"].cumprod()[5]
+    Out[2]: 5.960464477539062e+16
+
+We return incorrect results with the 6th value.
+
+*New Behavior*
+
+.. ipython:: python
+
+    df = pd.DataFrame({"key": ["b"] * 7, "value": 625})
+    df.groupby("key")["value"].cumprod()
+
+We overflow with the 7th value, but the 6th value is still correct.
 
 .. _whatsnew_160.notable_bug_fixes.notable_bug_fix2:
 
@@ -103,14 +130,17 @@ Deprecations
 
 Performance improvements
 ~~~~~~~~~~~~~~~~~~~~~~~~
-- Performance improvement in :meth:`.DataFrameGroupBy.median` and :meth:`.SeriesGroupBy.median` for nullable dtypes (:issue:`37493`)
+- Performance improvement in :meth:`.DataFrameGroupBy.median` and :meth:`.SeriesGroupBy.median` and :meth:`.GroupBy.cumprod` for nullable dtypes (:issue:`37493`)
 - Performance improvement in :meth:`MultiIndex.argsort` and :meth:`MultiIndex.sort_values` (:issue:`48406`)
 - Performance improvement in :meth:`MultiIndex.union` without missing values and without duplicates (:issue:`48505`)
 - Performance improvement in :meth:`.DataFrameGroupBy.mean`, :meth:`.SeriesGroupBy.mean`, :meth:`.DataFrameGroupBy.var`, and :meth:`.SeriesGroupBy.var` for extension array dtypes (:issue:`37493`)
+- Performance improvement in :meth:`MultiIndex.isin` when ``level=None`` (:issue:`48622`)
 - Performance improvement for :meth:`Series.value_counts` with nullable dtype (:issue:`48338`)
 - Performance improvement for :class:`Series` constructor passing integer numpy array with nullable dtype (:issue:`48338`)
+- Performance improvement in :func:`merge` and :meth:`DataFrame.join` when joining on a sorted :class:`MultiIndex` (:issue:`48504`)
 - Performance improvement in :meth:`DataFrame.loc` and :meth:`Series.loc` for tuple-based indexing of a :class:`MultiIndex` (:issue:`48384`)
 - Performance improvement for :meth:`MultiIndex.unique` (:issue:`48335`)
+- Performance improvement for :meth:`MultiIndex.intersection` (:issue:`48604`)
 - Performance improvement in ``var`` for nullable dtypes (:issue:`48379`).
 - Performance improvement to :func:`read_sas` with ``blank_missing=True`` (:issue:`48502`)
 -
@@ -150,6 +180,7 @@ Conversion
 ^^^^^^^^^^
 - Bug in constructing :class:`Series` with ``int64`` dtype from a string list raising instead of casting (:issue:`44923`)
 - Bug in :meth:`DataFrame.eval` incorrectly raising an ``AttributeError`` when there are negative values in function call (:issue:`46471`)
+- Bug where any :class:`ExtensionDtype` subclass with ``kind="M"`` would be interpreted as a timezone type (:issue:`34986`)
 -
 
 Strings
@@ -175,9 +206,12 @@ Missing
 
 MultiIndex
 ^^^^^^^^^^
+- Bug in :class:`MultiIndex.set_levels` raising ``IndexError`` when setting empty level (:issue:`48636`)
 - Bug in :meth:`MultiIndex.unique` losing extension array dtype (:issue:`48335`)
+- Bug in :meth:`MultiIndex.intersection` losing extension array (:issue:`48604`)
 - Bug in :meth:`MultiIndex.union` losing extension array (:issue:`48498`, :issue:`48505`)
 - Bug in :meth:`MultiIndex.append` not checking names for equality (:issue:`48288`)
+- Bug in :meth:`MultiIndex.symmetric_difference` losing extension array (:issue:`48607`)
 -
 
 I/O