API: warning to raise KeyError in the future if not all elements of a list are selected via .loc

closes #15747

Author: jreback

pandas/core/indexing.py

@@ -1416,12 +1416,32 @@ def _has_valid_type(self, key, axis):
             if isinstance(key, tuple) and isinstance(ax, MultiIndex):
                 return True
 
-            # TODO: don't check the entire key unless necessary
-            if (not is_iterator(key) and len(key) and
-                    np.all(ax.get_indexer_for(key) < 0)):
+            if not is_iterator(key) and len(key):
 
-                raise KeyError("None of [%s] are in the [%s]" %
-                               (key, self.obj._get_axis_name(axis)))
+                # True indicates missing values
+                missing = ax.get_indexer_for(key) < 0
+
+                if np.any(missing):
+                    if len(key) == 1 or np.all(missing):
+                        raise KeyError("None of [%s] are in the [%s]" %
+                                       (key, self.obj._get_axis_name(axis)))
+
+                    else:
+
+                        # we skip the warning on Categorical/Interval
+                        # as this check is actually done (check for
+                        # non-missing values), but a bit later in the
+                        # code, so we want to avoid warning & then
+                        # just raising
+                        if not (ax.is_categorical() or ax.is_interval()):
+                            warnings.warn(
+                                "passing list-likes to .loc with "
+                                "any non-matching elements will raise\n"
+                                "KeyError in the future, "
+                                "you can use .reindex() as an alternative",
+                                FutureWarning, stacklevel=5)
+
+                return True
 
             return True
 

pandas/io/formats/excel.py

@@ -353,7 +353,15 @@ def __init__(self, df, na_rep='', float_format=None, cols=None,
             self.styler = None
         self.df = df
         if cols is not None:
-            self.df = df.loc[:, cols]
+
+            # all missing, raise
+            if not len(Index(cols) & df.columns):
+                raise KeyError
+
+            # 1 missing is ok
+            # TODO(jreback)k this should raise
+            # on *any* missing columns
+            self.df = df.reindex(columns=cols)
         self.columns = self.df.columns
         self.float_format = float_format
         self.index = index

pandas/tests/indexing/test_categorical.py

@@ -111,7 +111,8 @@ def test_loc_listlike(self):
         assert_frame_equal(result, expected, check_index_type=True)
 
         # not all labels in the categories
-        pytest.raises(KeyError, lambda: self.df2.loc[['a', 'd']])
+        with pytest.raises(KeyError):
+            self.df2.loc[['a', 'd']]
 
     def test_loc_listlike_dtypes(self):
         # GH 11586

pandas/tests/indexing/test_datetime.py

@@ -223,7 +223,7 @@ def test_series_partial_set_datetime(self):
                 Timestamp('2011-01-03')]
         exp = Series([np.nan, 0.2, np.nan],
                      index=pd.DatetimeIndex(keys, name='idx'), name='s')
-        tm.assert_series_equal(ser.loc[keys], exp, check_index_type=True)
+        tm.assert_series_equal(ser.reindex(keys), exp, check_index_type=True)
 
     def test_series_partial_set_period(self):
         # GH 11497
@@ -248,5 +248,5 @@ def test_series_partial_set_period(self):
                 pd.Period('2011-01-03', freq='D')]
         exp = Series([np.nan, 0.2, np.nan],
                      index=pd.PeriodIndex(keys, name='idx'), name='s')
-        result = ser.loc[keys]
+        result = ser.reindex(keys)
         tm.assert_series_equal(result, exp)

pandas/tests/indexing/test_iloc.py

@@ -588,7 +588,8 @@ def test_iloc_non_unique_indexing(self):
         expected = DataFrame(new_list)
         expected = pd.concat([expected, DataFrame(index=idx[idx > sidx.max()])
                               ])
-        result = df2.loc[idx]
+        with tm.assert_produces_warning(FutureWarning, check_stacklevel=False):
+            result = df2.loc[idx]
         tm.assert_frame_equal(result, expected, check_index_type=False)
 
     def test_iloc_empty_list_indexer_is_ok(self):

pandas/tests/indexing/test_indexing.py

@@ -176,7 +176,8 @@ def test_dups_fancy_indexing(self):
              'test1': [7., 6, np.nan],
              'other': ['d', 'c', np.nan]}, index=rows)
 
-        result = df.loc[rows]
+        with tm.assert_produces_warning(FutureWarning, check_stacklevel=False):
+            result = df.loc[rows]
         tm.assert_frame_equal(result, expected)
 
         # see GH5553, make sure we use the right indexer
@@ -186,7 +187,8 @@ def test_dups_fancy_indexing(self):
                               'other': [np.nan, np.nan, np.nan,
                                         'd', 'c', np.nan]},
                              index=rows)
-        result = df.loc[rows]
+        with tm.assert_produces_warning(FutureWarning, check_stacklevel=False):
+            result = df.loc[rows]
         tm.assert_frame_equal(result, expected)
 
         # inconsistent returns for unique/duplicate indices when values are
@@ -203,20 +205,23 @@ def test_dups_fancy_indexing(self):
 
         # GH 4619; duplicate indexer with missing label
         df = DataFrame({"A": [0, 1, 2]})
-        result = df.loc[[0, 8, 0]]
+        with tm.assert_produces_warning(FutureWarning, check_stacklevel=False):
+            result = df.loc[[0, 8, 0]]
         expected = DataFrame({"A": [0, np.nan, 0]}, index=[0, 8, 0])
         tm.assert_frame_equal(result, expected, check_index_type=False)
 
         df = DataFrame({"A": list('abc')})
-        result = df.loc[[0, 8, 0]]
+        with tm.assert_produces_warning(FutureWarning, check_stacklevel=False):
+            result = df.loc[[0, 8, 0]]
         expected = DataFrame({"A": ['a', np.nan, 'a']}, index=[0, 8, 0])
         tm.assert_frame_equal(result, expected, check_index_type=False)
 
         # non unique with non unique selector
         df = DataFrame({'test': [5, 7, 9, 11]}, index=['A', 'A', 'B', 'C'])
         expected = DataFrame(
             {'test': [5, 7, 5, 7, np.nan]}, index=['A', 'A', 'A', 'A', 'E'])
-        result = df.loc[['A', 'A', 'E']]
+        with tm.assert_produces_warning(FutureWarning, check_stacklevel=False):
+            result = df.loc[['A', 'A', 'E']]
         tm.assert_frame_equal(result, expected)
 
         # GH 5835
@@ -227,7 +232,8 @@ def test_dups_fancy_indexing(self):
         expected = pd.concat(
             [df.loc[:, ['A', 'B']], DataFrame(np.nan, columns=['C'],
                                               index=df.index)], axis=1)
-        result = df.loc[:, ['A', 'B', 'C']]
+        with tm.assert_produces_warning(FutureWarning, check_stacklevel=False):
+            result = df.loc[:, ['A', 'B', 'C']]
         tm.assert_frame_equal(result, expected)
 
         # GH 6504, multi-axis indexing

pandas/tests/indexing/test_loc.py

@@ -152,12 +152,15 @@ def test_loc_getitem_label_list(self):
                           [Timestamp('20130102'), Timestamp('20130103')],
                           typs=['ts'], axes=0)
 
+    def test_loc_getitem_label_list_with_missing(self):
         self.check_result('list lbl', 'loc', [0, 1, 2], 'indexer', [0, 1, 2],
                           typs=['empty'], fails=KeyError)
-        self.check_result('list lbl', 'loc', [0, 2, 3], 'ix', [0, 2, 3],
-                          typs=['ints', 'uints'], axes=0, fails=KeyError)
-        self.check_result('list lbl', 'loc', [3, 6, 7], 'ix', [3, 6, 7],
-                          typs=['ints', 'uints'], axes=1, fails=KeyError)
+        with tm.assert_produces_warning(FutureWarning, check_stacklevel=False):
+            self.check_result('list lbl', 'loc', [0, 2, 3], 'ix', [0, 2, 3],
+                              typs=['ints', 'uints'], axes=0, fails=KeyError)
+        with tm.assert_produces_warning(FutureWarning, check_stacklevel=False):
+            self.check_result('list lbl', 'loc', [3, 6, 7], 'ix', [3, 6, 7],
+                              typs=['ints', 'uints'], axes=1, fails=KeyError)
         self.check_result('list lbl', 'loc', [4, 8, 10], 'ix', [4, 8, 10],
                           typs=['ints', 'uints'], axes=2, fails=KeyError)
 
@@ -249,7 +252,7 @@ def test_loc_to_fail(self):
         pytest.raises(KeyError, lambda: s.loc[['4']])
 
         s.loc[-1] = 3
-        result = s.loc[[-1, -2]]
+        result = s.reindex([-1, -2])
         expected = Series([3, np.nan], index=[-1, -2])
         tm.assert_series_equal(result, expected)
 
@@ -277,6 +280,23 @@ def f():
 
         pytest.raises(KeyError, f)
 
+    def test_loc_getitem_list_with_fail(self):
+        # 15747
+        # should KeyError if *any* missing labels
+
+        s = Series([1, 2, 3])
+
+        s.loc[[2]]
+
+        with pytest.raises(KeyError):
+            s.loc[[3]]
+
+        # a non-match and a match
+        with tm.assert_produces_warning(FutureWarning):
+            expected = s.loc[[2, 3]]
+        result = s.reindex([2, 3])
+        tm.assert_series_equal(result, expected)
+
     def test_loc_getitem_label_slice(self):
 
         # label slices (with ints)

pandas/tests/indexing/test_partial.py

@@ -222,52 +222,93 @@ def test_series_partial_set(self):
         # Regression from GH4825
         ser = Series([0.1, 0.2], index=[1, 2])
 
-        # loc
+        # loc equiv to .reindex
         expected = Series([np.nan, 0.2, np.nan], index=[3, 2, 3])
-        result = ser.loc[[3, 2, 3]]
+        with tm.assert_produces_warning(FutureWarning, check_stacklevel=False):
+            result = ser.loc[[3, 2, 3]]
+        tm.assert_series_equal(result, expected, check_index_type=True)
+
+        result = ser.reindex([3, 2, 3])
         tm.assert_series_equal(result, expected, check_index_type=True)
 
         expected = Series([np.nan, 0.2, np.nan, np.nan], index=[3, 2, 3, 'x'])
-        result = ser.loc[[3, 2, 3, 'x']]
+        with tm.assert_produces_warning(FutureWarning, check_stacklevel=False):
+            result = ser.loc[[3, 2, 3, 'x']]
+        tm.assert_series_equal(result, expected, check_index_type=True)
+
+        result = ser.reindex([3, 2, 3, 'x'])
         tm.assert_series_equal(result, expected, check_index_type=True)
 
         expected = Series([0.2, 0.2, 0.1], index=[2, 2, 1])
         result = ser.loc[[2, 2, 1]]
         tm.assert_series_equal(result, expected, check_index_type=True)
 
         expected = Series([0.2, 0.2, np.nan, 0.1], index=[2, 2, 'x', 1])
-        result = ser.loc[[2, 2, 'x', 1]]
+        with tm.assert_produces_warning(FutureWarning, check_stacklevel=False):
+            result = ser.loc[[2, 2, 'x', 1]]
+        tm.assert_series_equal(result, expected, check_index_type=True)
+
+        result = ser.reindex([2, 2, 'x', 1])
         tm.assert_series_equal(result, expected, check_index_type=True)
 
         # raises as nothing in in the index
         pytest.raises(KeyError, lambda: ser.loc[[3, 3, 3]])
 
         expected = Series([0.2, 0.2, np.nan], index=[2, 2, 3])
-        result = ser.loc[[2, 2, 3]]
+        with tm.assert_produces_warning(FutureWarning, check_stacklevel=False):
+            result = ser.loc[[2, 2, 3]]
         tm.assert_series_equal(result, expected, check_index_type=True)
 
+        result = ser.reindex([2, 2, 3])
+        tm.assert_series_equal(result, expected, check_index_type=True)
+
+        s = Series([0.1, 0.2, 0.3], index=[1, 2, 3])
         expected = Series([0.3, np.nan, np.nan], index=[3, 4, 4])
-        result = Series([0.1, 0.2, 0.3], index=[1, 2, 3]).loc[[3, 4, 4]]
+        with tm.assert_produces_warning(FutureWarning, check_stacklevel=False):
+            result = s.loc[[3, 4, 4]]
         tm.assert_series_equal(result, expected, check_index_type=True)
 
+        result = s.reindex([3, 4, 4])
+        tm.assert_series_equal(result, expected, check_index_type=True)
+
+        s = Series([0.1, 0.2, 0.3, 0.4],
+                   index=[1, 2, 3, 4])
         expected = Series([np.nan, 0.3, 0.3], index=[5, 3, 3])
-        result = Series([0.1, 0.2, 0.3, 0.4],
-                        index=[1, 2, 3, 4]).loc[[5, 3, 3]]
+        with tm.assert_produces_warning(FutureWarning, check_stacklevel=False):
+            result = s.loc[[5, 3, 3]]
+        tm.assert_series_equal(result, expected, check_index_type=True)
+
+        result = s.reindex([5, 3, 3])
         tm.assert_series_equal(result, expected, check_index_type=True)
 
+        s = Series([0.1, 0.2, 0.3, 0.4],
+                   index=[1, 2, 3, 4])
         expected = Series([np.nan, 0.4, 0.4], index=[5, 4, 4])
-        result = Series([0.1, 0.2, 0.3, 0.4],
-                        index=[1, 2, 3, 4]).loc[[5, 4, 4]]
+        with tm.assert_produces_warning(FutureWarning, check_stacklevel=False):
+            result = s.loc[[5, 4, 4]]
+        tm.assert_series_equal(result, expected, check_index_type=True)
+
+        result = s.reindex([5, 4, 4])
         tm.assert_series_equal(result, expected, check_index_type=True)
 
+        s = Series([0.1, 0.2, 0.3, 0.4],
+                   index=[4, 5, 6, 7])
         expected = Series([0.4, np.nan, np.nan], index=[7, 2, 2])
-        result = Series([0.1, 0.2, 0.3, 0.4],
-                        index=[4, 5, 6, 7]).loc[[7, 2, 2]]
+        with tm.assert_produces_warning(FutureWarning, check_stacklevel=False):
+            result = s.loc[[7, 2, 2]]
         tm.assert_series_equal(result, expected, check_index_type=True)
 
+        result = s.reindex([7, 2, 2])
+        tm.assert_series_equal(result, expected, check_index_type=True)
+
+        s = Series([0.1, 0.2, 0.3, 0.4],
+                   index=[1, 2, 3, 4])
         expected = Series([0.4, np.nan, np.nan], index=[4, 5, 5])
-        result = Series([0.1, 0.2, 0.3, 0.4],
-                        index=[1, 2, 3, 4]).loc[[4, 5, 5]]
+        with tm.assert_produces_warning(FutureWarning, check_stacklevel=False):
+            result = s.loc[[4, 5, 5]]
+        tm.assert_series_equal(result, expected, check_index_type=True)
+
+        result = s.reindex([4, 5, 5])
         tm.assert_series_equal(result, expected, check_index_type=True)
 
         # iloc
@@ -284,13 +325,15 @@ def test_series_partial_set_with_name(self):
         # loc
         exp_idx = Index([3, 2, 3], dtype='int64', name='idx')
         expected = Series([np.nan, 0.2, np.nan], index=exp_idx, name='s')
-        result = ser.loc[[3, 2, 3]]
+        with tm.assert_produces_warning(FutureWarning, check_stacklevel=False):
+            result = ser.loc[[3, 2, 3]]
         tm.assert_series_equal(result, expected, check_index_type=True)
 
         exp_idx = Index([3, 2, 3, 'x'], dtype='object', name='idx')
         expected = Series([np.nan, 0.2, np.nan, np.nan], index=exp_idx,
                           name='s')
-        result = ser.loc[[3, 2, 3, 'x']]
+        with tm.assert_produces_warning(FutureWarning, check_stacklevel=False):
+            result = ser.loc[[3, 2, 3, 'x']]
         tm.assert_series_equal(result, expected, check_index_type=True)
 
         exp_idx = Index([2, 2, 1], dtype='int64', name='idx')
@@ -300,49 +343,58 @@ def test_series_partial_set_with_name(self):
 
         exp_idx = Index([2, 2, 'x', 1], dtype='object', name='idx')
         expected = Series([0.2, 0.2, np.nan, 0.1], index=exp_idx, name='s')
-        result = ser.loc[[2, 2, 'x', 1]]
+        with tm.assert_produces_warning(FutureWarning, check_stacklevel=False):
+            result = ser.loc[[2, 2, 'x', 1]]
         tm.assert_series_equal(result, expected, check_index_type=True)
 
         # raises as nothing in in the index
         pytest.raises(KeyError, lambda: ser.loc[[3, 3, 3]])
 
         exp_idx = Index([2, 2, 3], dtype='int64', name='idx')
         expected = Series([0.2, 0.2, np.nan], index=exp_idx, name='s')
-        result = ser.loc[[2, 2, 3]]
+        with tm.assert_produces_warning(FutureWarning, check_stacklevel=False):
+            result = ser.loc[[2, 2, 3]]
         tm.assert_series_equal(result, expected, check_index_type=True)
 
         exp_idx = Index([3, 4, 4], dtype='int64', name='idx')
         expected = Series([0.3, np.nan, np.nan], index=exp_idx, name='s')
         idx = Index([1, 2, 3], dtype='int64', name='idx')
-        result = Series([0.1, 0.2, 0.3], index=idx, name='s').loc[[3, 4, 4]]
+        with tm.assert_produces_warning(FutureWarning, check_stacklevel=False):
+            result = Series([0.1, 0.2, 0.3],
+                            index=idx,
+                            name='s').loc[[3, 4, 4]]
         tm.assert_series_equal(result, expected, check_index_type=True)
 
         exp_idx = Index([5, 3, 3], dtype='int64', name='idx')
         expected = Series([np.nan, 0.3, 0.3], index=exp_idx, name='s')
         idx = Index([1, 2, 3, 4], dtype='int64', name='idx')
-        result = Series([0.1, 0.2, 0.3, 0.4], index=idx,
-                        name='s').loc[[5, 3, 3]]
+        with tm.assert_produces_warning(FutureWarning, check_stacklevel=False):
+            result = Series([0.1, 0.2, 0.3, 0.4], index=idx,
+                            name='s').loc[[5, 3, 3]]
         tm.assert_series_equal(result, expected, check_index_type=True)
 
         exp_idx = Index([5, 4, 4], dtype='int64', name='idx')
         expected = Series([np.nan, 0.4, 0.4], index=exp_idx, name='s')
         idx = Index([1, 2, 3, 4], dtype='int64', name='idx')
-        result = Series([0.1, 0.2, 0.3, 0.4], index=idx,
-                        name='s').loc[[5, 4, 4]]
+        with tm.assert_produces_warning(FutureWarning, check_stacklevel=False):
+            result = Series([0.1, 0.2, 0.3, 0.4], index=idx,
+                            name='s').loc[[5, 4, 4]]
         tm.assert_series_equal(result, expected, check_index_type=True)
 
         exp_idx = Index([7, 2, 2], dtype='int64', name='idx')
         expected = Series([0.4, np.nan, np.nan], index=exp_idx, name='s')
         idx = Index([4, 5, 6, 7], dtype='int64', name='idx')
-        result = Series([0.1, 0.2, 0.3, 0.4], index=idx,
-                        name='s').loc[[7, 2, 2]]
+        with tm.assert_produces_warning(FutureWarning, check_stacklevel=False):
+            result = Series([0.1, 0.2, 0.3, 0.4], index=idx,
+                            name='s').loc[[7, 2, 2]]
         tm.assert_series_equal(result, expected, check_index_type=True)
 
         exp_idx = Index([4, 5, 5], dtype='int64', name='idx')
         expected = Series([0.4, np.nan, np.nan], index=exp_idx, name='s')
         idx = Index([1, 2, 3, 4], dtype='int64', name='idx')
-        result = Series([0.1, 0.2, 0.3, 0.4], index=idx,
-                        name='s').loc[[4, 5, 5]]
+        with tm.assert_produces_warning(FutureWarning, check_stacklevel=False):
+            result = Series([0.1, 0.2, 0.3, 0.4], index=idx,
+                            name='s').loc[[4, 5, 5]]
         tm.assert_series_equal(result, expected, check_index_type=True)
 
         # iloc