Merge branch 'main' into fix/resample-interpolate-fails-with-inplace-true-58690-remove-inplace-option

Author: cbpygit

ci/code_checks.sh

@@ -471,7 +471,6 @@ if [[ -z "$CHECK" || "$CHECK" == "docstrings" ]]; then
         -i "pandas.plotting.andrews_curves RT03,SA01" \
         -i "pandas.plotting.lag_plot RT03,SA01" \
         -i "pandas.plotting.scatter_matrix PR07,SA01" \
-        -i "pandas.qcut PR07,SA01" \
         -i "pandas.set_eng_float_format RT03,SA01" \
         -i "pandas.testing.assert_extension_array_equal SA01" \
         -i "pandas.tseries.offsets.BDay PR02,SA01" \

doc/source/getting_started/intro_tutorials/09_timeseries.rst

@@ -295,7 +295,7 @@ Aggregate the current hourly time series values to the monthly maximum value in
 
 .. ipython:: python
 
-    monthly_max = no_2.resample("ME").max()
+    monthly_max = no_2.resample("MS").max()
     monthly_max
 
 A very powerful method on time series data with a datetime index, is the

doc/source/user_guide/10min.rst

@@ -101,7 +101,7 @@ truncated for brevity.
 Viewing data
 ------------
 
-See the :ref:`Essentially basics functionality section <basics>`.
+See the :ref:`Essential basic functionality section <basics>`.
 
 Use :meth:`DataFrame.head` and :meth:`DataFrame.tail` to view the top and bottom rows of the frame
 respectively:

doc/source/user_guide/boolean.rst

@@ -37,6 +37,19 @@ If you would prefer to keep the ``NA`` values you can manually fill them with ``
 
    s[mask.fillna(True)]
 
+If you create a column of ``NA`` values (for example to fill them later)
+with ``df['new_col'] = pd.NA``, the ``dtype`` would be set to ``object`` in the
+new column. The performance on this column will be worse than with
+the appropriate type. It's better to use
+``df['new_col'] = pd.Series(pd.NA, dtype="boolean")``
+(or another ``dtype`` that supports ``NA``).
+
+.. ipython:: python
+
+   df = pd.DataFrame()
+   df['objects'] = pd.NA
+   df.dtypes
+
 .. _boolean.kleene:
 
 Kleene logical operations

doc/source/user_guide/integer_na.rst

@@ -84,6 +84,19 @@ with the dtype.
    In the future, we may provide an option for :class:`Series` to infer a
    nullable-integer dtype.
 
+If you create a column of ``NA`` values (for example to fill them later)
+with ``df['new_col'] = pd.NA``, the ``dtype`` would be set to ``object`` in the
+new column. The performance on this column will be worse than with
+the appropriate type. It's better to use
+``df['new_col'] = pd.Series(pd.NA, dtype="Int64")``
+(or another ``dtype`` that supports ``NA``).
+
+.. ipython:: python
+
+   df = pd.DataFrame()
+   df['objects'] = pd.NA
+   df.dtypes
+
 Operations
 ----------
 

doc/source/user_guide/timeseries.rst

@@ -1864,15 +1864,15 @@ to resample based on datetimelike column in the frame, it can passed to the
        ),
    )
    df
-   df.resample("ME", on="date")[["a"]].sum()
+   df.resample("MS", on="date")[["a"]].sum()
 
 Similarly, if you instead want to resample by a datetimelike
 level of ``MultiIndex``, its name or location can be passed to the
 ``level`` keyword.
 
 .. ipython:: python
 
-   df.resample("ME", level="d")[["a"]].sum()
+   df.resample("MS", level="d")[["a"]].sum()
 
 .. _timeseries.iterating-label:
 

doc/source/whatsnew/v3.0.0.rst

@@ -42,6 +42,7 @@ Other enhancements
 - :meth:`DataFrame.corrwith` now accepts ``min_periods`` as optional arguments, as in :meth:`DataFrame.corr` and :meth:`Series.corr` (:issue:`9490`)
 - :meth:`DataFrame.cummin`, :meth:`DataFrame.cummax`, :meth:`DataFrame.cumprod` and :meth:`DataFrame.cumsum` methods now have a ``numeric_only`` parameter (:issue:`53072`)
 - :meth:`DataFrame.fillna` and :meth:`Series.fillna` can now accept ``value=None``; for non-object dtype the corresponding NA value will be used (:issue:`57723`)
+- :meth:`DataFrame.pivot_table` and :func:`pivot_table` now allow the passing of keyword arguments to ``aggfunc`` through ``**kwargs`` (:issue:`57884`)
 - :meth:`Series.cummin` and :meth:`Series.cummax` now supports :class:`CategoricalDtype` (:issue:`52335`)
 - :meth:`Series.plot` now correctly handle the ``ylabel`` parameter for pie charts, allowing for explicit control over the y-axis label (:issue:`58239`)
 - Restore support for reading Stata 104-format and enable reading 103-format dta files (:issue:`58554`)
@@ -280,6 +281,34 @@ Other Deprecations
 
 Removal of prior version deprecations/changes
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Enforced deprecation of aliases ``M``, ``Q``, ``Y``, etc. in favour of ``ME``, ``QE``, ``YE``, etc. for offsets
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+Renamed the following offset aliases (:issue:`57986`):
+
++-------------------------------+------------------+------------------+
+| offset                        | removed alias    | new alias        |
++===============================+==================+==================+
+|:class:`MonthEnd`              |      ``M``       |     ``ME``       |
++-------------------------------+------------------+------------------+
+|:class:`BusinessMonthEnd`      |      ``BM``      |     ``BME``      |
++-------------------------------+------------------+------------------+
+|:class:`SemiMonthEnd`          |      ``SM``      |     ``SME``      |
++-------------------------------+------------------+------------------+
+|:class:`CustomBusinessMonthEnd`|      ``CBM``     |     ``CBME``     |
++-------------------------------+------------------+------------------+
+|:class:`QuarterEnd`            |      ``Q``       |     ``QE``       |
++-------------------------------+------------------+------------------+
+|:class:`BQuarterEnd`           |      ``BQ``      |     ``BQE``      |
++-------------------------------+------------------+------------------+
+|:class:`YearEnd`               |      ``Y``       |     ``YE``       |
++-------------------------------+------------------+------------------+
+|:class:`BYearEnd`              |      ``BY``      |     ``BYE``      |
++-------------------------------+------------------+------------------+
+
+Other Removals
+^^^^^^^^^^^^^^
 - :class:`.DataFrameGroupBy.idxmin`, :class:`.DataFrameGroupBy.idxmax`, :class:`.SeriesGroupBy.idxmin`, and :class:`.SeriesGroupBy.idxmax` will now raise a ``ValueError`` when used with ``skipna=False`` and an NA value is encountered (:issue:`10694`)
 - :func:`concat` no longer ignores empty objects when determining output dtypes (:issue:`39122`)
 - :func:`concat` with all-NA entries no longer ignores the dtype of those entries when determining the result dtype (:issue:`40893`)
@@ -343,7 +372,7 @@ Removal of prior version deprecations/changes
 - Enforced deprecation of string ``A`` denoting frequency in :class:`YearEnd` and strings ``A-DEC``, ``A-JAN``, etc. denoting annual frequencies with various fiscal year ends (:issue:`57699`)
 - Enforced deprecation of string ``BAS`` denoting frequency in :class:`BYearBegin` and strings ``BAS-DEC``, ``BAS-JAN``, etc. denoting annual frequencies with various fiscal year starts (:issue:`57793`)
 - Enforced deprecation of string ``BA`` denoting frequency in :class:`BYearEnd` and strings ``BA-DEC``, ``BA-JAN``, etc. denoting annual frequencies with various fiscal year ends (:issue:`57793`)
-- Enforced deprecation of strings ``T``, ``L``, ``U``, and ``N`` denoting frequencies in :class:`Minute`, :class:`Second`, :class:`Milli`, :class:`Micro`, :class:`Nano` (:issue:`57627`)
+- Enforced deprecation of strings ``T``, ``L``, ``U``, and ``N`` denoting frequencies in :class:`Minute`, :class:`Milli`, :class:`Micro`, :class:`Nano` (:issue:`57627`)
 - Enforced deprecation of strings ``T``, ``L``, ``U``, and ``N`` denoting units in :class:`Timedelta` (:issue:`57627`)
 - Enforced deprecation of the behavior of :func:`concat` when ``len(keys) != len(objs)`` would truncate to the shorter of the two. Now this raises a ``ValueError`` (:issue:`43485`)
 - Enforced deprecation of values "pad", "ffill", "bfill", and "backfill" for :meth:`Series.interpolate` and :meth:`DataFrame.interpolate` (:issue:`57869`)
@@ -453,6 +482,7 @@ Categorical
 
 Datetimelike
 ^^^^^^^^^^^^
+- Bug in :attr:`is_year_start` where a DateTimeIndex constructed via a date_range with frequency 'MS' wouldn't have the correct year or quarter start attributes (:issue:`57377`)
 - Bug in :class:`Timestamp` constructor failing to raise when ``tz=None`` is explicitly specified in conjunction with timezone-aware ``tzinfo`` or data (:issue:`48688`)
 - Bug in :func:`date_range` where the last valid timestamp would sometimes not be produced (:issue:`56134`)
 - Bug in :func:`date_range` where using a negative frequency value would not include all points between the start and end values (:issue:`56382`)

pandas/_libs/tslibs/dtypes.pxd

@@ -12,9 +12,10 @@ cdef NPY_DATETIMEUNIT get_supported_reso(NPY_DATETIMEUNIT reso)
 cdef bint is_supported_unit(NPY_DATETIMEUNIT reso)
 
 cdef dict c_OFFSET_TO_PERIOD_FREQSTR
-cdef dict c_OFFSET_DEPR_FREQSTR
-cdef dict c_REVERSE_OFFSET_DEPR_FREQSTR
+cdef dict c_PERIOD_TO_OFFSET_FREQSTR
+cdef dict c_OFFSET_RENAMED_FREQSTR
 cdef dict c_DEPR_ABBREVS
+cdef dict c_PERIOD_AND_OFFSET_DEPR_FREQSTR
 cdef dict attrname_to_abbrevs
 cdef dict npy_unit_to_attrname
 cdef dict attrname_to_npy_unit

pandas/_libs/tslibs/dtypes.pyx

@@ -176,6 +176,10 @@ OFFSET_TO_PERIOD_FREQSTR: dict = {
     "EOM": "M",
     "BME": "M",
     "SME": "M",
+    "BMS": "M",
+    "CBME": "M",
+    "CBMS": "M",
+    "SMS": "M",
     "BQS": "Q",
     "QS": "Q",
     "BQE": "Q",
@@ -228,7 +232,6 @@ OFFSET_TO_PERIOD_FREQSTR: dict = {
     "YE-NOV": "Y-NOV",
     "W": "W",
     "ME": "M",
-    "Y": "Y",
     "BYE": "Y",
     "BYE-DEC": "Y-DEC",
     "BYE-JAN": "Y-JAN",
@@ -245,7 +248,7 @@ OFFSET_TO_PERIOD_FREQSTR: dict = {
     "YS": "Y",
     "BYS": "Y",
 }
-cdef dict c_OFFSET_DEPR_FREQSTR = {
+cdef dict c_OFFSET_RENAMED_FREQSTR = {
     "M": "ME",
     "Q": "QE",
     "Q-DEC": "QE-DEC",
@@ -303,10 +306,37 @@ cdef dict c_OFFSET_DEPR_FREQSTR = {
     "BQ-OCT": "BQE-OCT",
     "BQ-NOV": "BQE-NOV",
 }
-cdef dict c_OFFSET_TO_PERIOD_FREQSTR = OFFSET_TO_PERIOD_FREQSTR
-cdef dict c_REVERSE_OFFSET_DEPR_FREQSTR = {
-    v: k for k, v in c_OFFSET_DEPR_FREQSTR.items()
+PERIOD_TO_OFFSET_FREQSTR = {
+    "M": "ME",
+    "Q": "QE",
+    "Q-DEC": "QE-DEC",
+    "Q-JAN": "QE-JAN",
+    "Q-FEB": "QE-FEB",
+    "Q-MAR": "QE-MAR",
+    "Q-APR": "QE-APR",
+    "Q-MAY": "QE-MAY",
+    "Q-JUN": "QE-JUN",
+    "Q-JUL": "QE-JUL",
+    "Q-AUG": "QE-AUG",
+    "Q-SEP": "QE-SEP",
+    "Q-OCT": "QE-OCT",
+    "Q-NOV": "QE-NOV",
+    "Y": "YE",
+    "Y-DEC": "YE-DEC",
+    "Y-JAN": "YE-JAN",
+    "Y-FEB": "YE-FEB",
+    "Y-MAR": "YE-MAR",
+    "Y-APR": "YE-APR",
+    "Y-MAY": "YE-MAY",
+    "Y-JUN": "YE-JUN",
+    "Y-JUL": "YE-JUL",
+    "Y-AUG": "YE-AUG",
+    "Y-SEP": "YE-SEP",
+    "Y-OCT": "YE-OCT",
+    "Y-NOV": "YE-NOV",
 }
+cdef dict c_OFFSET_TO_PERIOD_FREQSTR = OFFSET_TO_PERIOD_FREQSTR
+cdef dict c_PERIOD_TO_OFFSET_FREQSTR = PERIOD_TO_OFFSET_FREQSTR
 
 # Map deprecated resolution abbreviations to correct resolution abbreviations
 cdef dict c_DEPR_ABBREVS = {
@@ -316,6 +346,11 @@ cdef dict c_DEPR_ABBREVS = {
     "S": "s",
 }
 
+cdef dict c_PERIOD_AND_OFFSET_DEPR_FREQSTR = {
+    "w": "W",
+    "MIN": "min",
+}
+
 
 class FreqGroup(Enum):
     # Mirrors c_FreqGroup in the .pxd file

pandas/_libs/tslibs/fields.pyx

@@ -253,9 +253,10 @@ def get_start_end_field(
         # month of year. Other offsets use month, startingMonth as ending
         # month of year.
 
-        if freq_name.lstrip("B")[0:2] in ["MS", "QS", "YS"]:
+        if freq_name.lstrip("B")[0:2] in ["QS", "YS"]:
             end_month = 12 if month_kw == 1 else month_kw - 1
             start_month = month_kw
+
         else:
             end_month = month_kw
             start_month = (end_month % 12) + 1

pandas/_libs/tslibs/offsets.pyx

@@ -57,8 +57,10 @@ from pandas._libs.tslibs.ccalendar cimport (
 from pandas._libs.tslibs.conversion cimport localize_pydatetime
 from pandas._libs.tslibs.dtypes cimport (
     c_DEPR_ABBREVS,
-    c_OFFSET_DEPR_FREQSTR,
-    c_REVERSE_OFFSET_DEPR_FREQSTR,
+    c_OFFSET_RENAMED_FREQSTR,
+    c_OFFSET_TO_PERIOD_FREQSTR,
+    c_PERIOD_AND_OFFSET_DEPR_FREQSTR,
+    c_PERIOD_TO_OFFSET_FREQSTR,
     periods_per_day,
 )
 from pandas._libs.tslibs.nattype cimport (
@@ -4711,6 +4713,34 @@ INVALID_FREQ_ERR_MSG = "Invalid frequency: {0}"
 _offset_map = {}
 
 
+def _validate_to_offset_alias(alias: str, is_period: bool) -> None:
+    if not is_period:
+        if alias.upper() in c_OFFSET_RENAMED_FREQSTR:
+            raise ValueError(
+                f"\'{alias}\' is no longer supported for offsets. Please "
+                f"use \'{c_OFFSET_RENAMED_FREQSTR.get(alias.upper())}\' "
+                f"instead."
+            )
+        if (alias.upper() != alias and
+                alias.lower() not in {"s", "ms", "us", "ns"} and
+                alias.upper().split("-")[0].endswith(("S", "E"))):
+            raise ValueError(INVALID_FREQ_ERR_MSG.format(alias))
+    if (is_period and
+            alias.upper() in c_OFFSET_TO_PERIOD_FREQSTR and
+            alias != "ms" and
+            alias.upper().split("-")[0].endswith(("S", "E"))):
+        if (alias.upper().startswith("B") or
+                alias.upper().startswith("S") or
+                alias.upper().startswith("C")):
+            raise ValueError(INVALID_FREQ_ERR_MSG.format(alias))
+        else:
+            alias_msg = "".join(alias.upper().split("E", 1))
+            raise ValueError(
+                f"for Period, please use \'{alias_msg}\' "
+                f"instead of \'{alias}\'"
+            )
+
+
 # TODO: better name?
 def _get_offset(name: str) -> BaseOffset:
     """
@@ -4850,54 +4880,26 @@ cpdef to_offset(freq, bint is_period=False):
 
             tups = zip(split[0::4], split[1::4], split[2::4])
             for n, (sep, stride, name) in enumerate(tups):
-                if not is_period and name.upper() in c_OFFSET_DEPR_FREQSTR:
-                    warnings.warn(
-                        f"\'{name}\' is deprecated and will be removed "
-                        f"in a future version, please use "
-                        f"\'{c_OFFSET_DEPR_FREQSTR.get(name.upper())}\' instead.",
-                        FutureWarning,
-                        stacklevel=find_stack_level(),
-                    )
-                    name = c_OFFSET_DEPR_FREQSTR[name.upper()]
-                if (not is_period and
-                        name != name.upper() and
-                        name.lower() not in {"s", "ms", "us", "ns"} and
-                        name.upper().split("-")[0].endswith(("S", "E"))):
-                    warnings.warn(
-                        f"\'{name}\' is deprecated and will be removed "
-                        f"in a future version, please use "
-                        f"\'{name.upper()}\' instead.",
-                        FutureWarning,
-                        stacklevel=find_stack_level(),
-                    )
-                    name = name.upper()
-                if is_period and name.upper() in c_REVERSE_OFFSET_DEPR_FREQSTR:
-                    if name.upper().startswith("Y"):
-                        raise ValueError(
-                            f"for Period, please use \'Y{name.upper()[2:]}\' "
-                            f"instead of \'{name}\'"
-                        )
-                    if (name.upper().startswith("B") or
-                            name.upper().startswith("S") or
-                            name.upper().startswith("C")):
-                        raise ValueError(INVALID_FREQ_ERR_MSG.format(name))
-                    else:
-                        raise ValueError(
-                            f"for Period, please use "
-                            f"\'{c_REVERSE_OFFSET_DEPR_FREQSTR.get(name.upper())}\' "
-                            f"instead of \'{name}\'"
-                        )
-                elif is_period and name.upper() in c_OFFSET_DEPR_FREQSTR:
-                    if name.upper() != name:
+                _validate_to_offset_alias(name, is_period)
+                if is_period:
+                    if name.upper() in c_PERIOD_TO_OFFSET_FREQSTR:
+                        if name.upper() != name:
+                            raise ValueError(
+                                f"\'{name}\' is no longer supported, "
+                                f"please use \'{name.upper()}\' instead.",
+                            )
+                        name = c_PERIOD_TO_OFFSET_FREQSTR.get(name.upper())
+
+                    if name in c_PERIOD_AND_OFFSET_DEPR_FREQSTR:
                         warnings.warn(
-                            f"\'{name}\' is deprecated and will be removed in "
-                            f"a future version, please use \'{name.upper()}\' "
-                            f"instead.",
+                            f"\'{name}\' is deprecated and will be removed "
+                            f"in a future version, please use "
+                            f"\'{c_PERIOD_AND_OFFSET_DEPR_FREQSTR.get(name)}\' "
+                            f" instead.",
                             FutureWarning,
                             stacklevel=find_stack_level(),
-                        )
-                    name = c_OFFSET_DEPR_FREQSTR.get(name.upper())
-
+                            )
+                        name = c_PERIOD_AND_OFFSET_DEPR_FREQSTR.get(name)
                 if sep != "" and not sep.isspace():
                     raise ValueError("separator must be spaces")
                 prefix = _lite_rule_alias.get(name) or name

pandas/core/arrays/datetimes.py

@@ -143,7 +143,7 @@ def f(self):
                 month_kw = 12
                 if freq:
                     kwds = freq.kwds
-                    month_kw = kwds.get("startingMonth", kwds.get("month", 12))
+                    month_kw = kwds.get("startingMonth", kwds.get("month", month_kw))
 
                 if freq is not None:
                     freq_name = freq.name