Merge remote-tracking branch 'upstream/master' into am-more-tests

Author: jorisvandenbossche

.github/workflows/ci.yml

@@ -150,42 +150,44 @@ jobs:
       uses: ./.github/actions/setup
 
     - name: Run tests
+      env:
+        PANDAS_DATA_MANAGER: array
       run: |
         source activate pandas-dev
-        pytest pandas/tests/frame/methods --array-manager
-        pytest pandas/tests/frame/test_constructors.py --array-manager
-        pytest pandas/tests/frame/constructors/ --array-manager
-        pytest pandas/tests/frame/test_reductions.py --array-manager
-        pytest pandas/tests/reductions/ --array-manager
-        pytest pandas/tests/generic/test_generic.py --array-manager
-        pytest pandas/tests/arithmetic/ --array-manager
-        pytest pandas/tests/groupby/ --array-manager
-        pytest pandas/tests/resample/ --array-manager
-        pytest pandas/tests/reshape/merge --array-manager
-
-        pytest pandas/tests/series/methods --array-manager
-        pytest pandas/tests/series/test_* --array-manager
+        pytest pandas/tests/frame/methods
+        pytest pandas/tests/frame/test_constructors.py
+        pytest pandas/tests/frame/constructors/
+        pytest pandas/tests/frame/test_reductions.py
+        pytest pandas/tests/reductions/
+        pytest pandas/tests/generic/test_generic.py
+        pytest pandas/tests/arithmetic/
+        pytest pandas/tests/groupby/
+        pytest pandas/tests/resample/
+        pytest pandas/tests/reshape/merge
+
+        pytest pandas/tests/series/methods
+        pytest pandas/tests/series/test_*
 
         # indexing subset (temporary since other tests don't pass yet)
-        pytest pandas/tests/frame/indexing/test_indexing.py::TestDataFrameIndexing::test_setitem_boolean --array-manager
-        pytest pandas/tests/frame/indexing/test_where.py --array-manager
-        pytest pandas/tests/frame/indexing/test_setitem.py::TestDataFrameSetItem::test_setitem_multi_index --array-manager
-        pytest pandas/tests/frame/indexing/test_setitem.py::TestDataFrameSetItem::test_setitem_listlike_indexer_duplicate_columns --array-manager
-        pytest pandas/tests/indexing/multiindex/test_setitem.py::TestMultiIndexSetItem::test_astype_assignment_with_dups --array-manager
-        pytest pandas/tests/indexing/multiindex/test_setitem.py::TestMultiIndexSetItem::test_frame_setitem_multi_column --array-manager
-
-        pytest pandas/tests/api/ --array-manager
-        pytest pandas/tests/base/ --array-manager
-        pytest pandas/tests/computation/ --array-manager
-        pytest pandas/tests/config/ --array-manager
-        pytest pandas/tests/dtypes/ --array-manager
-        pytest pandas/tests/generic/ --array-manager
-        pytest pandas/tests/indexes/ --array-manager
-        pytest pandas/tests/libs/ --array-manager
-        pytest pandas/tests/plotting/ --array-manager
-        pytest pandas/tests/scalar/ --array-manager
-        pytest pandas/tests/strings/ --array-manager
-        pytest pandas/tests/tools/ --array-manager
-        pytest pandas/tests/tseries/ --array-manager
-        pytest pandas/tests/tslibs/ --array-manager
-        pytest pandas/tests/util/ --array-manager
+        pytest pandas/tests/frame/indexing/test_indexing.py::TestDataFrameIndexing::test_setitem_boolean
+        pytest pandas/tests/frame/indexing/test_where.py
+        pytest pandas/tests/frame/indexing/test_setitem.py::TestDataFrameSetItem::test_setitem_multi_index
+        pytest pandas/tests/frame/indexing/test_setitem.py::TestDataFrameSetItem::test_setitem_listlike_indexer_duplicate_columns
+        pytest pandas/tests/indexing/multiindex/test_setitem.py::TestMultiIndexSetItem::test_astype_assignment_with_dups
+        pytest pandas/tests/indexing/multiindex/test_setitem.py::TestMultiIndexSetItem::test_frame_setitem_multi_column
+
+        pytest pandas/tests/api/
+        pytest pandas/tests/base/
+        pytest pandas/tests/computation/
+        pytest pandas/tests/config/
+        pytest pandas/tests/dtypes/
+        pytest pandas/tests/generic/
+        pytest pandas/tests/indexes/
+        pytest pandas/tests/libs/
+        pytest pandas/tests/plotting/
+        pytest pandas/tests/scalar/
+        pytest pandas/tests/strings/
+        pytest pandas/tests/tools/
+        pytest pandas/tests/tseries/
+        pytest pandas/tests/tslibs/
+        pytest pandas/tests/util/

pandas/conftest.py

@@ -100,19 +100,6 @@ def pytest_addoption(parser):
         action="store_true",
         help="Fail if a test is skipped for missing data file.",
     )
-    parser.addoption(
-        "--array-manager",
-        "--am",
-        action="store_true",
-        help="Use the experimental ArrayManager as default data manager.",
-    )
-
-
-def pytest_sessionstart(session):
-    # Note: we need to set the option here and not in pytest_runtest_setup below
-    # to ensure this is run before creating fixture data
-    if session.config.getoption("--array-manager"):
-        pd.options.mode.data_manager = "array"
 
 
 def pytest_runtest_setup(item):

pandas/core/config_init.py

@@ -9,6 +9,7 @@
 module is imported, register them here rather than in the module.
 
 """
+import os
 import warnings
 
 import pandas._config.config as cf
@@ -483,18 +484,32 @@ def use_inf_as_na_cb(key):
     cf.register_option(
         "use_inf_as_null", False, use_inf_as_null_doc, cb=use_inf_as_na_cb
     )
-    cf.register_option(
-        "data_manager",
-        "block",
-        "Internal data manager type",
-        validator=is_one_of_factory(["block", "array"]),
-    )
+
 
 cf.deprecate_option(
     "mode.use_inf_as_null", msg=use_inf_as_null_doc, rkey="mode.use_inf_as_na"
 )
 
 
+data_manager_doc = """
+: string
+    Internal data manager type; can be "block" or "array". Defaults to "block",
+    unless overridden by the 'PANDAS_DATA_MANAGER' environment variable (needs
+    to be set before pandas is imported).
+"""
+
+
+with cf.config_prefix("mode"):
+    cf.register_option(
+        "data_manager",
+        # Get the default from an environment variable, if set, otherwise defaults
+        # to "block". This environment variable can be set for testing.
+        os.environ.get("PANDAS_DATA_MANAGER", "block"),
+        data_manager_doc,
+        validator=is_one_of_factory(["block", "array"]),
+    )
+
+
 # user warnings
 chained_assignment = """
 : string

pandas/tests/apply/test_frame_apply.py

@@ -37,12 +37,6 @@ def test_apply(float_frame):
         assert result[d] == expected
         assert result.index is float_frame.index
 
-    # invalid axis
-    df = DataFrame([[1, 2, 3], [4, 5, 6], [7, 8, 9]], index=["a", "a", "c"])
-    msg = "No axis named 2 for object type DataFrame"
-    with pytest.raises(ValueError, match=msg):
-        df.apply(lambda x: x, 2)
-
     # GH 9573
     df = DataFrame({"c0": ["A", "A", "B", "B"], "c1": ["C", "C", "D", "D"]})
     result = df.apply(lambda ts: ts.astype("category"))
@@ -579,9 +573,6 @@ def test_applymap_na_ignore(float_frame):
     strlen_frame_with_na[mask] = pd.NA
     tm.assert_frame_equal(strlen_frame_na_ignore, strlen_frame_with_na)
 
-    with pytest.raises(ValueError, match="na_action must be .*Got 'abc'"):
-        float_frame_with_na.applymap(lambda x: len(str(x)), na_action="abc")
-
 
 def test_applymap_box_timestamps():
     # GH 2689, GH 2627

pandas/tests/apply/test_invalid_arg.py

@@ -6,12 +6,14 @@
 #     4. invalid result shape/type
 # If your test does not fit into one of these categories, add to this list.
 
+from itertools import chain
 import re
 
 import numpy as np
 import pytest
 
 from pandas import (
+    Categorical,
     DataFrame,
     Series,
     date_range,
@@ -34,6 +36,19 @@ def test_result_type_error(result_type, int_frame_const_col):
         df.apply(lambda x: [1, 2, 3], axis=1, result_type=result_type)
 
 
+def test_apply_invalid_axis_value():
+    df = DataFrame([[1, 2, 3], [4, 5, 6], [7, 8, 9]], index=["a", "a", "c"])
+    msg = "No axis named 2 for object type DataFrame"
+    with pytest.raises(ValueError, match=msg):
+        df.apply(lambda x: x, 2)
+
+
+def test_applymap_invalid_na_action(float_frame):
+    # GH 23803
+    with pytest.raises(ValueError, match="na_action must be .*Got 'abc'"):
+        float_frame.applymap(lambda x: len(str(x)), na_action="abc")
+
+
 def test_agg_raises():
     # GH 26513
     df = DataFrame({"A": [0, 1], "B": [1, 2]})
@@ -43,6 +58,28 @@ def test_agg_raises():
         df.agg()
 
 
+def test_map_with_invalid_na_action_raises():
+    # https://github.com/pandas-dev/pandas/issues/32815
+    s = Series([1, 2, 3])
+    msg = "na_action must either be 'ignore' or None"
+    with pytest.raises(ValueError, match=msg):
+        s.map(lambda x: x, na_action="____")
+
+
+def test_map_categorical_na_action():
+    values = Categorical(list("ABBABCD"), categories=list("DCBA"), ordered=True)
+    s = Series(values, name="XX", index=list("abcdefg"))
+    with pytest.raises(NotImplementedError, match=tm.EMPTY_STRING_PATTERN):
+        s.map(lambda x: x, na_action="ignore")
+
+
+def test_map_datetimetz_na_action():
+    values = date_range("2011-01-01", "2011-01-02", freq="H").tz_localize("Asia/Tokyo")
+    s = Series(values, name="XX")
+    with pytest.raises(NotImplementedError, match=tm.EMPTY_STRING_PATTERN):
+        s.map(lambda x: x, na_action="ignore")
+
+
 @pytest.mark.parametrize("box", [DataFrame, Series])
 @pytest.mark.parametrize("method", ["apply", "agg", "transform"])
 @pytest.mark.parametrize("func", [{"A": {"B": "sum"}}, {"A": {"B": ["sum"]}}])
@@ -54,6 +91,22 @@ def test_nested_renamer(box, method, func):
         getattr(obj, method)(func)
 
 
+def test_series_agg_nested_renamer():
+    s = Series(range(6), dtype="int64", name="series")
+    msg = "nested renamer is not supported"
+    with pytest.raises(SpecificationError, match=msg):
+        s.agg({"foo": ["min", "max"]})
+
+
+def test_multiple_aggregators_with_dict_api():
+
+    s = Series(range(6), dtype="int64", name="series")
+    # nested renaming
+    msg = "nested renamer is not supported"
+    with pytest.raises(SpecificationError, match=msg):
+        s.agg({"foo": ["min", "max"], "bar": ["sum", "mean"]})
+
+
 def test_transform_nested_renamer():
     # GH 35964
     match = "nested renamer is not supported"
@@ -208,13 +261,37 @@ def transform2(row):
         DataFrame([["a", "b"], ["b", "a"]]), [["cumprod", TypeError]]
     ),
 )
-def test_agg_cython_table_raises(df, func, expected, axis):
+def test_agg_cython_table_raises_frame(df, func, expected, axis):
     # GH 21224
     msg = "can't multiply sequence by non-int of type 'str'"
     with pytest.raises(expected, match=msg):
         df.agg(func, axis=axis)
 
 
+@pytest.mark.parametrize(
+    "series, func, expected",
+    chain(
+        tm.get_cython_table_params(
+            Series("a b c".split()),
+            [
+                ("mean", TypeError),  # mean raises TypeError
+                ("prod", TypeError),
+                ("std", TypeError),
+                ("var", TypeError),
+                ("median", TypeError),
+                ("cumprod", TypeError),
+            ],
+        )
+    ),
+)
+def test_agg_cython_table_raises_series(series, func, expected):
+    # GH21224
+    msg = r"[Cc]ould not convert|can't multiply sequence by non-int of type"
+    with pytest.raises(expected, match=msg):
+        # e.g. Series('a b'.split()).cumprod() will raise
+        series.agg(func)
+
+
 def test_transform_none_to_type():
     # GH#34377
     df = DataFrame({"a": [None]})

pandas/tests/apply/test_series_apply.py

@@ -20,7 +20,6 @@
     timedelta_range,
 )
 import pandas._testing as tm
-from pandas.core.base import SpecificationError
 
 
 def test_series_map_box_timedelta():
@@ -269,20 +268,6 @@ def test_demo():
     expected = Series([0], index=["foo"], name="series")
     tm.assert_series_equal(result, expected)
 
-    # nested renaming
-    msg = "nested renamer is not supported"
-    with pytest.raises(SpecificationError, match=msg):
-        s.agg({"foo": ["min", "max"]})
-
-
-def test_multiple_aggregators_with_dict_api():
-
-    s = Series(range(6), dtype="int64", name="series")
-    # nested renaming
-    msg = "nested renamer is not supported"
-    with pytest.raises(SpecificationError, match=msg):
-        s.agg({"foo": ["min", "max"], "bar": ["sum", "mean"]})
-
 
 def test_agg_apply_evaluate_lambdas_the_same(string_series):
     # test that we are evaluating row-by-row first
@@ -439,30 +424,6 @@ def test_agg_cython_table_transform(series, func, expected):
     tm.assert_series_equal(result, expected)
 
 
-@pytest.mark.parametrize(
-    "series, func, expected",
-    chain(
-        tm.get_cython_table_params(
-            Series("a b c".split()),
-            [
-                ("mean", TypeError),  # mean raises TypeError
-                ("prod", TypeError),
-                ("std", TypeError),
-                ("var", TypeError),
-                ("median", TypeError),
-                ("cumprod", TypeError),
-            ],
-        )
-    ),
-)
-def test_agg_cython_table_raises(series, func, expected):
-    # GH21224
-    msg = r"[Cc]ould not convert|can't multiply sequence by non-int of type"
-    with pytest.raises(expected, match=msg):
-        # e.g. Series('a b'.split()).cumprod() will raise
-        series.agg(func)
-
-
 def test_series_apply_no_suffix_index():
     # GH36189
     s = Series([4] * 3)
@@ -732,9 +693,6 @@ def test_map_categorical():
     tm.assert_series_equal(result, exp)
     assert result.dtype == object
 
-    with pytest.raises(NotImplementedError, match=tm.EMPTY_STRING_PATTERN):
-        s.map(lambda x: x, na_action="ignore")
-
 
 def test_map_datetimetz():
     values = pd.date_range("2011-01-01", "2011-01-02", freq="H").tz_localize(
@@ -756,9 +714,6 @@ def test_map_datetimetz():
     exp = Series(list(range(24)) + [0], name="XX", dtype=np.int64)
     tm.assert_series_equal(result, exp)
 
-    with pytest.raises(NotImplementedError, match=tm.EMPTY_STRING_PATTERN):
-        s.map(lambda x: x, na_action="ignore")
-
     # not vectorized
     def f(x):
         if not isinstance(x, pd.Timestamp):
@@ -827,14 +782,6 @@ def test_map_float_to_string_precision():
     assert result == expected
 
 
-def test_map_with_invalid_na_action_raises():
-    # https://github.com/pandas-dev/pandas/issues/32815
-    s = Series([1, 2, 3])
-    msg = "na_action must either be 'ignore' or None"
-    with pytest.raises(ValueError, match=msg):
-        s.map(lambda x: x, na_action="____")
-
-
 def test_apply_to_timedelta():
     list_of_valid_strings = ["00:00:01", "00:00:02"]
     a = pd.to_timedelta(list_of_valid_strings)