BUG: fix issue with sparse concatting

This was originally brought up in :issue:`18686` and :issue:`18914`.
Basically the problem is when you use get_dummies with sparse=True it
will return a SparseDataFrame with sparse and dense columns. This is in
fact not what we want. What we want is a DataFrame with sparse and dense
columns.

Inside of pandas.core.dtypes.concat is a function that defines the
factory class which needed to be changed.

Author: hexgnu

doc/source/whatsnew/v0.23.0.txt

@@ -363,6 +363,7 @@ Reshaping
 - Bug in :func:`Series.rank` where ``Series`` containing ``NaT`` modifies the ``Series`` inplace (:issue:`18521`)
 - Bug in :func:`cut` which fails when using readonly arrays (:issue:`18773`)
 - Bug in :func:`Dataframe.pivot_table` which fails when the ``aggfunc`` arg is of type string.  The behavior is now consistent with other methods like ``agg`` and ``apply`` (:issue:`18713`)
+- Bug in :func:`pandas.core.dtypes.concat._get_series_result_type` which returns SparseDataFrame even if not all contained in Frame are sparse. (:issue:`18914` and :issue:`18686`)
 
 
 Numeric

pandas/core/dtypes/concat.py

@@ -89,10 +89,10 @@ def _get_series_result_type(result, objs=None):
 def _get_frame_result_type(result, objs):
     """
     return appropriate class of DataFrame-like concat
-    if any block is SparseBlock, return SparseDataFrame
+    if all blocks are SparseBlock, return SparseDataFrame
     otherwise, return 1st obj
     """
-    if any(b.is_sparse for b in result.blocks):
+    if result.blocks and all(b.is_sparse for b in result.blocks):
         from pandas.core.sparse.api import SparseDataFrame
         return SparseDataFrame
     else:

pandas/core/sparse/series.py

@@ -168,7 +168,6 @@ def __init__(self, data=None, index=None, sparse_index=None, kind='block',
                 if index is None:
                     index = data.index.view()
                 else:
-
                     data = data.reindex(index, copy=False)
 
             else:

pandas/tests/reshape/test_reshape.py

@@ -454,6 +454,14 @@ def test_dataframe_dummies_preserve_categorical_dtype(self, dtype):
 
             tm.assert_frame_equal(result, expected)
 
+    def test_get_dummies_dont_sparsify_all_columns(self):
+        # GH18914
+        df = DataFrame.from_items([('GDP', [1, 2]), ('Nation', ['AB', 'CD'])])
+        df = get_dummies(df, columns=['Nation'], sparse=True)
+        df2 = df.reindex(columns=['GDP'])
+
+        tm.assert_index_equal(df.index, df2.index)
+
 
 class TestCategoricalReshape(object):
 

pandas/tests/sparse/test_combine_concat.py

@@ -344,10 +344,17 @@ def test_concat_sparse_dense(self):
 
         res = pd.concat([self.dense3, sparse], axis=1)
         exp = pd.concat([self.dense3, self.dense1], axis=1)
-        assert isinstance(res, pd.SparseDataFrame)
+        # See GH18914 and #18686 for why this should be 
+        # A DataFrame
+        assert isinstance(res, pd.DataFrame)
+        for column in self.dense3.columns:
+            tm.assert_series_equal(res[column], exp[column])
+
         tm.assert_frame_equal(res, exp)
 
         res = pd.concat([sparse, self.dense3], axis=1)
         exp = pd.concat([self.dense1, self.dense3], axis=1)
-        assert isinstance(res, pd.SparseDataFrame)
+        assert isinstance(res, pd.DataFrame)
+        for column in self.dense3.columns:
+            tm.assert_series_equal(res[column], exp[column])
         tm.assert_frame_equal(res, exp)