fixup

Author: kernc

pandas/core/indexing.py

@@ -526,10 +526,7 @@ def setter(item, v):
                     # set the item, possibly having a dtype change
                     s._consolidate_inplace()
                     s = s.copy()
-                    # if is_sparse(s):
-                    #     s.set_value(pi, v, takeable=is_list_like(pi))
-                    # else:
-                    s._data = s._data.setitem(indexer=pi, value=v)  # TODO THIS USE THIS
+                    s._data = s._data.setitem(indexer=pi, value=v)
                     s._maybe_update_cacher(clear=True)
 
                 # reset the sliced object if unique
@@ -634,13 +631,8 @@ def can_do_equal_len():
 
             # actually do the set
             self.obj._consolidate_inplace()
-            if is_sparse(self.obj):
-                # SparseSeries has underlying SparseArray, which doesn't
-                # support resizing
-                self.obj[indexer] = value
-            else:
-                self.obj._data = self.obj._data.setitem(indexer=indexer,
-                                                        value=value)
+            self.obj._data = self.obj._data.setitem(indexer=indexer,
+                                                    value=value)
             self.obj._maybe_update_cacher(clear=True)
 
     def _align_series(self, indexer, ser, multiindex_indexer=False):

pandas/core/internals.py

@@ -1768,6 +1768,8 @@ def putmask(self, mask, new, align=True, inplace=False, axis=0,
         # Cannot modify a SparseArray
         if is_sparse(new_values):
             new_values = new_values.to_dense()
+            if is_sparse(mask):
+                mask = mask.to_dense()
 
         if isinstance(new, np.ndarray) and len(new) == len(mask):
             new = new[mask]
@@ -2812,8 +2814,7 @@ def _astype(self, dtype, copy=False, errors='raise', values=None,
                                           placement=self.mgr_locs)
 
     def _can_hold_element(self, element):
-        return np.issubdtype(np.asanyarray(element).dtype,
-                             self.sp_values.dtype)
+        return np.can_cast(np.asarray(element).dtype, self.sp_values.dtype)
 
     def _try_coerce_result(self, result):
         if np.ndim(result) > 0 and not is_sparse(result):

pandas/core/series.py

@@ -949,8 +949,7 @@ def set_value(self, label, value, takeable=False):
 
         Returns
         -------
-        series : Series
-            self
+        self : Series
         """
         warnings.warn("set_value is deprecated and will be removed "
                       "in a future release. Please use "

pandas/core/sparse/array.py

@@ -36,6 +36,7 @@
 import pandas.core.algorithms as algos
 import pandas.core.ops as ops
 import pandas.io.formats.printing as printing
+from pandas.errors import PerformanceWarning
 from pandas.util._decorators import Appender
 from pandas.core.indexes.base import _index_shared_docs
 
@@ -398,27 +399,39 @@ def set_values(self, indexer, value):
         """
         # If indexer is not a single int position, easiest to handle via dense
         if not is_scalar(indexer):
+            warnings.warn(
+                'Setting SparseSeries/Array values is particularly '
+                'inefficient when indexing with multiple keys because the '
+                'whole series series is made dense interim.',
+                PerformanceWarning, stacklevel=2)
+
             values = self.to_dense()
             values[indexer] = value
             return SparseArray(values, kind=self.kind,
                                fill_value=self.fill_value)
 
+        warnings.warn(
+            'Setting SparseSeries/Array values is inefficient '
+            '(a copy of data is made)', PerformanceWarning, stacklevel=2)
+
         # If label already in sparse index, just switch the value on a copy
         idx = self.sp_index.lookup(indexer)
         if idx != -1:
             obj = self.copy()
             obj.sp_values[idx] = value
             return obj
 
+        # Otherwise, construct a new array, and insert the new value in the
+        # correct position
         indices = self.sp_index.to_int_index().indices
         pos = np.searchsorted(indices, indexer)
 
         indices = np.insert(indices, pos, indexer)
         sp_values = np.insert(self.sp_values, pos, value)
+        sp_index = _make_index(self.sp_index.length, indices, self.kind)
 
-        return SparseArray(
-            sp_values, fill_value=self.fill_value,
-            sparse_index=_make_index(self.sp_index.length, indices, self.kind))
+        return SparseArray(sp_values, sparse_index=sp_index,
+                           fill_value=self.fill_value)
 
     def to_dense(self, fill=None):
         """

pandas/core/sparse/frame.py

@@ -465,29 +465,6 @@ def _get_value(self, index, col, takeable=False):
         return series._get_value(index, takeable=takeable)
     _get_value.__doc__ = get_value.__doc__
 
-    def set_value(self, index, col, value, takeable=False):
-        """
-        Put single value at passed column and index
-
-        Parameters
-        ----------
-        index : row label
-        col : column label
-        value : scalar value
-        takeable : interpret the index/col as indexers, default False
-
-        Notes
-        -----
-        This method *always* returns a new object for backwards compatibility.
-
-        Returns
-        -------
-        frame : DataFrame
-        """
-        self = self.copy(deep=False)
-        return super(SparseDataFrame, self).set_value(index, col, value,
-                                                      takeable=takeable)
-
     def _slice(self, slobj, axis=0, kind=None):
         if axis == 0:
             new_index = self.index[slobj]

pandas/core/sparse/series.py

@@ -10,12 +10,10 @@
 
 from pandas.core.dtypes.missing import isna, notna
 from pandas.core.dtypes.common import is_scalar, is_sparse
-from pandas.core.common import _values_from_object, _maybe_match_name, \
-    PerformanceWarning
+from pandas.core.common import _values_from_object, _maybe_match_name
 
 from pandas.compat.numpy import function as nv
 from pandas.core.index import Index, _ensure_index, InvalidIndexError
-from pandas.core.indexing import check_bool_indexer
 from pandas.core.series import Series
 from pandas.core.frame import DataFrame
 from pandas.core.internals import SingleBlockManager
@@ -317,7 +315,7 @@ def __array_wrap__(self, result, context=None):
         else:
             fill_value = self.fill_value
 
-        # Results size unchanged, old sparse index is valid ???
+        # If result size matches, old sparse index is valid ???
         if np.size(result) == self.sp_index.npoints:
             sp_index = self.sp_index
         else:
@@ -434,22 +432,7 @@ def _get_values(self, indexer):
             return self[indexer]
 
     def _set_with_engine(self, key, value):
-<<<<<<< HEAD
         return self._set_value(key, value)
-||||||| parent of e95270f1e... ENH: Allow SparseDataFrame/SparseSeries values assignment
-        return self.set_value(key, value)
-=======
-        takeable = False
-
-        # Sparse doesn't support reshaping so the standard .where() does
-        # not apply. We short-circuit bool indexers here by treating them as
-        # regular list of indexes and setting each array/value separately
-        if com.is_bool_indexer(key):
-            key = check_bool_indexer(self.index, key).nonzero()[0]
-            takeable = True
-
-        return self.set_value(key, value, takeable=takeable)
->>>>>>> e95270f1e... ENH: Allow SparseDataFrame/SparseSeries values assignment
 
     def abs(self):
         """
@@ -539,7 +522,7 @@ def set_value(self, label, value, takeable=False):
 
         Returns
         -------
-        series : SparseSeries
+        self : SparseSeries
         """
         warnings.warn("set_value is deprecated and will be removed "
                       "in a future release. Please use "
@@ -548,62 +531,7 @@ def set_value(self, label, value, takeable=False):
         return self._set_value(label, value, takeable=takeable)
 
     def _set_value(self, label, value, takeable=False):
-        try:
-            loc = self.index.get_loc(label)
-        except (KeyError, TypeError):
-            loc = None
-
-        warnings.warn(
-            'Setting SparseSeries values is inefficient '
-            '(a copy of data is made)', PerformanceWarning, stacklevel=2)
-
-        # If label is not unique in index, or it is takeable,
-        # amend the series by amending its dense copy
-        if not isinstance(loc, int) or takeable:
-            warnings.warn(
-                'Setting SparseSeries values is particularly inefficient when '
-                'indexing with a non-unique label because the whole series '
-                'is made dense interim.', PerformanceWarning, stacklevel=2)
-            values = self.to_dense()
-            values.set_value(label, value, takeable=takeable)
-
-            index = values.index
-            sp_index = None
-            values = values.to_sparse(kind=self.kind,
-                                      fill_value=self.fill_value)
-
-        # If label is unique key and not takeable, then it is more space-
-        # efficient to not make the whole series dense, rather just its
-        # defined part
-        else:
-            values = self._to_dense(sparse_only=True)
-            old_index = values.index
-            values.set_value(label, value, takeable=takeable)
-            index = self.index
-
-            # label was already in sparse index, we can just reuse old index
-            if label in old_index:
-                sp_index = self.sp_index
-
-            # label might have been at least in .index
-            else:
-                # and if not, just add it, then construct both indexes anew
-                if loc is None:
-                    index = self.index.append(Index((label,)))
-                    loc = len(index) - 1
-
-                indices = np.append(
-                    self.sp_index.to_int_index().indices,
-                    np.array(loc, dtype=np.int32))
-                order = indices.argsort()
-                values = values.values.take(order)
-                indices = indices.take(order)
-                sp_index = _make_index(len(index), indices, self.kind)
-
-        values = SparseArray(values, sparse_index=sp_index, kind=self.kind,
-                             fill_value=self.fill_value)
-        self._data = SingleBlockManager(values, index)
-        self._index = index
+        self._data = self._data.copy().setitem(indexer=label, value=value)
         return self
     _set_value.__doc__ = set_value.__doc__
 
@@ -643,23 +571,6 @@ def to_dense(self, sparse_only=False):
             warnings.warn(("The 'sparse_only' parameter has been deprecated "
                            "and will be removed in a future version."),
                           FutureWarning, stacklevel=2)
-        return self._to_dense(sparse_only=sparse_only)
-
-    def _to_dense(self, sparse_only=False):
-        """
-        Convert SparseSeries to a Series.
-
-        Parameters
-        ----------
-        sparse_only: bool, default False
-            If True, return just the non-sparse values, or the dense version
-            of `self.values` if False.
-
-        Returns
-        -------
-        s : Series
-        """
-        if sparse_only:
             int_index = self.sp_index.to_int_index()
             index = self.index.take(int_index.indices)
             return Series(self.sp_values, index=index, name=self.name)

pandas/tests/sparse/test_frame.py

@@ -1536,31 +1536,49 @@ def test_quantile_multi(self):
         tm.assert_sp_frame_equal(result, sparse_expected)
 
 
-@pytest.mark.parametrize('kind', ['integer', 'block'])
-@pytest.mark.parametrize('indexer', [None, 'loc', 'iloc', 'at', 'iat'])
-@pytest.mark.parametrize('key', [0, [0, 1], [True, False], None])
-def test_frame_assignment(kind, indexer, key):
-    try_multiple = 'at' not in (indexer or '')
-    is_multi_key = np.asarray(key).ndim > 0
-    if is_multi_key and not try_multiple:
-        return
-    if not indexer and not is_multi_key and key is not None:  # skip non-multikey with setitem
-        return
-    if indexer and key is None:  # skip df indexer with non-setitem
-        return
-
+def _test_assignment(kind, indexer, key=None):
     arr = np.array([[1, nan],
                     [nan, 1]])
+    df = DataFrame(arr, copy=True)
     sdf = SparseDataFrame(arr, default_kind=kind).to_sparse(kind=kind)
 
+    def get_indexer(df):
+        return getattr(df, indexer) if indexer else df
+
     if key is None:
         key = pd.isnull(sdf).to_sparse()
 
-    arr = arr.copy()
-    arr[np.asarray(key)] = 2
-    res = SparseDataFrame(arr, default_kind=kind).to_sparse(kind=kind)
+    get_indexer(sdf)[key] = 2
 
-    sdf_setitem = getattr(sdf, indexer) if indexer else sdf
-    sdf_setitem[key] = 2
+    get_indexer(df)[key] = 2
+    res = df.to_sparse(kind=kind)
 
     tm.assert_sp_frame_equal(sdf, res)
+
+
+@pytest.fixture(params=['integer', 'block'])
+def spindex_kind(request):
+    return request.param
+
+
+@pytest.mark.parametrize('indexer', ['at', 'iat'])
+@pytest.mark.parametrize('key', [0])
+def test_frame_assignment_at(spindex_kind, indexer, key):
+    _test_assignment(spindex_kind, indexer, key)
+
+
+@pytest.mark.parametrize('indexer', ['loc', 'iloc'])
+@pytest.mark.parametrize('key', [0, [0, 1], [True, False]])
+def test_frame_assignment_loc(spindex_kind, indexer, key):
+    _test_assignment(spindex_kind, indexer, key)
+
+
+@pytest.mark.parametrize('key', [None, [0, 1], [True, False]])
+def test_frame_assignment_setitem(spindex_kind, key):
+    _test_assignment(spindex_kind, None, key)
+
+
+@pytest.mark.parametrize('indexer', ['loc', 'at'])
+@pytest.mark.parametrize('key', [3])
+def test_frame_assignment_extend_index(spindex_kind, indexer, key):
+    _test_assignment(spindex_kind, indexer, key)

pandas/tests/sparse/test_series.py

@@ -1541,11 +1541,11 @@ def test_constructor_dict_datetime64_index(datetime_type):
 @pytest.mark.parametrize('indexer', [None, 'loc', 'iloc', 'at', 'iat'])
 @pytest.mark.parametrize('key', [0, [0, 1], 2, [2, 3],
                                  [True, False, False, False],
-                                 [False, False, False, True],])
+                                 [False, False, False, True]])
 def test_series_assignment(kind, indexer, key):
-    try_multiple = 'at' not in (indexer or '')
-    is_multi_key = np.asarray(key).ndim > 0
-    if is_multi_key and not try_multiple:
+    is_multikey = np.asarray(key).ndim > 0
+    skip_multikey = 'at' in (indexer or '')
+    if is_multikey and skip_multikey:
         return
 
     arr = np.array([0., 0., nan, nan])