PERF: msgpack encoding changnes to use to/from string for speed boosts

API: disable sparse structure encodings and unicode indexes

Author: jreback

pandas/core/common.py

@@ -1871,8 +1871,12 @@ def _asarray_tuplesafe(values, dtype=None):
         else:
             # Making a 1D array that safely contains tuples is a bit tricky
             # in numpy, leading to the following
-            result = np.empty(len(values), dtype=object)
-            result[:] = values
+            try:
+                result = np.empty(len(values), dtype=object)
+                result[:] = values
+            except (ValueError):
+                # we have a list-of-list
+                result[:] = [ tuple(x) for x in values ]
 
     return result
 

pandas/io/packers.py

@@ -45,7 +45,7 @@
 
 import numpy as np
 from pandas import compat
-from pandas.compat import u
+from pandas.compat import u, PY3
 from pandas import (
     Timestamp, Period, Series, DataFrame, Panel, Panel4D,
     Index, MultiIndex, Int64Index, PeriodIndex, DatetimeIndex, Float64Index, NaT
@@ -168,6 +168,10 @@ def convert(values):
         values = values.view('i8')
     v = values.ravel()
 
+    # convert object
+    if dtype == np.object_:
+        return v.tolist()
+
     if compressor == 'zlib':
 
         # return string arrays like they are
@@ -189,12 +193,7 @@ def convert(values):
         return blosc.compress(v, typesize=dtype.itemsize)
 
     # ndarray (on original dtype)
-    if dtype == 'float64' or dtype == 'int64':
-        return v
-
-    # as a list
-    return v.tolist()
-
+    return v.tostring()
 
 def unconvert(values, dtype, compress=None):
 
@@ -216,9 +215,8 @@ def unconvert(values, dtype, compress=None):
 
         return np.frombuffer(values, dtype=dtype)
 
-    # as a list
-    return np.array(values, dtype=dtype)
-
+    # from a string
+    return np.fromstring(values.encode('latin1'),dtype=dtype)
 
 def encode(obj):
     """
@@ -253,19 +251,20 @@ def encode(obj):
                     'klass': obj.__class__.__name__,
                     'name': getattr(obj, 'name', None),
                     'dtype': obj.dtype.num,
-                    'data': obj.tolist()}
+                    'data': convert(obj.values)}
     elif isinstance(obj, Series):
         if isinstance(obj, SparseSeries):
-            d = {'typ': 'sparse_series',
-                 'klass': obj.__class__.__name__,
-                 'dtype': obj.dtype.num,
-                 'index': obj.index,
-                 'sp_index': obj.sp_index,
-                 'sp_values': convert(obj.sp_values),
-                 'compress': compressor}
-            for f in ['name', 'fill_value', 'kind']:
-                d[f] = getattr(obj, f, None)
-            return d
+            raise NotImplementedError("msgpack sparse series is not implemented")
+            #d = {'typ': 'sparse_series',
+            #     'klass': obj.__class__.__name__,
+            #     'dtype': obj.dtype.num,
+            #     'index': obj.index,
+            #     'sp_index': obj.sp_index,
+            #     'sp_values': convert(obj.sp_values),
+            #     'compress': compressor}
+            #for f in ['name', 'fill_value', 'kind']:
+            #    d[f] = getattr(obj, f, None)
+            #return d
         else:
             return {'typ': 'series',
                     'klass': obj.__class__.__name__,
@@ -276,23 +275,25 @@ def encode(obj):
                     'compress': compressor}
     elif issubclass(tobj, NDFrame):
         if isinstance(obj, SparseDataFrame):
-            d = {'typ': 'sparse_dataframe',
-                 'klass': obj.__class__.__name__,
-                 'columns': obj.columns}
-            for f in ['default_fill_value', 'default_kind']:
-                d[f] = getattr(obj, f, None)
-            d['data'] = dict([(name, ss)
-                             for name, ss in compat.iteritems(obj)])
-            return d
+            raise NotImplementedError("msgpack sparse frame is not implemented")
+            #d = {'typ': 'sparse_dataframe',
+            #     'klass': obj.__class__.__name__,
+            #     'columns': obj.columns}
+            #for f in ['default_fill_value', 'default_kind']:
+            #    d[f] = getattr(obj, f, None)
+            #d['data'] = dict([(name, ss)
+            #                 for name, ss in compat.iteritems(obj)])
+            #return d
         elif isinstance(obj, SparsePanel):
-            d = {'typ': 'sparse_panel',
-                 'klass': obj.__class__.__name__,
-                 'items': obj.items}
-            for f in ['default_fill_value', 'default_kind']:
-                d[f] = getattr(obj, f, None)
-            d['data'] = dict([(name, df)
-                             for name, df in compat.iteritems(obj)])
-            return d
+            raise NotImplementedError("msgpack sparse frame is not implemented")
+            #d = {'typ': 'sparse_panel',
+            #     'klass': obj.__class__.__name__,
+            #     'items': obj.items}
+            #for f in ['default_fill_value', 'default_kind']:
+            #    d[f] = getattr(obj, f, None)
+            #d['data'] = dict([(name, df)
+            #                 for name, df in compat.iteritems(obj)])
+            #return d
         else:
 
             data = obj._data
@@ -354,7 +355,7 @@ def encode(obj):
                 'klass': obj.__class__.__name__,
                 'indices': obj.indices,
                 'length': obj.length}
-    elif isinstance(obj, np.ndarray) and obj.dtype not in ['float64', 'int64']:
+    elif isinstance(obj, np.ndarray):
         return {'typ': 'ndarray',
                 'shape': obj.shape,
                 'ndim': obj.ndim,
@@ -394,14 +395,18 @@ def decode(obj):
         return Period(ordinal=obj['ordinal'], freq=obj['freq'])
     elif typ == 'index':
         dtype = dtype_for(obj['dtype'])
-        data = obj['data']
+        data = unconvert(obj['data'], np.typeDict[obj['dtype']], obj.get('compress'))
         return globals()[obj['klass']](data, dtype=dtype, name=obj['name'])
     elif typ == 'multi_index':
-        return globals()[obj['klass']].from_tuples(obj['data'], names=obj['names'])
+        data = unconvert(obj['data'], np.typeDict[obj['dtype']], obj.get('compress'))
+        data = [ tuple(x) for x in data ]
+        return globals()[obj['klass']].from_tuples(data, names=obj['names'])
     elif typ == 'period_index':
-        return globals()[obj['klass']](obj['data'], name=obj['name'], freq=obj['freq'])
+        data = unconvert(obj['data'], np.int64, obj.get('compress'))
+        return globals()[obj['klass']](data, name=obj['name'], freq=obj['freq'])
     elif typ == 'datetime_index':
-        return globals()[obj['klass']](obj['data'], freq=obj['freq'], tz=obj['tz'], name=obj['name'])
+        data = unconvert(obj['data'], np.int64, obj.get('compress'))
+        return globals()[obj['klass']](data, freq=obj['freq'], tz=obj['tz'], name=obj['name'])
     elif typ == 'series':
         dtype = dtype_for(obj['dtype'])
         index = obj['index']
@@ -425,17 +430,17 @@ def create_block(b):
         return timedelta(*obj['data'])
     elif typ == 'timedelta64':
         return np.timedelta64(int(obj['data']))
-    elif typ == 'sparse_series':
-        dtype = dtype_for(obj['dtype'])
-        return globals(
-        )[obj['klass']](unconvert(obj['sp_values'], dtype, obj['compress']), sparse_index=obj['sp_index'],
-                        index=obj['index'], fill_value=obj['fill_value'], kind=obj['kind'], name=obj['name'])
-    elif typ == 'sparse_dataframe':
-        return globals()[obj['klass']](obj['data'],
-                                       columns=obj['columns'], default_fill_value=obj['default_fill_value'], default_kind=obj['default_kind'])
-    elif typ == 'sparse_panel':
-        return globals()[obj['klass']](obj['data'],
-                                       items=obj['items'], default_fill_value=obj['default_fill_value'], default_kind=obj['default_kind'])
+    #elif typ == 'sparse_series':
+    #    dtype = dtype_for(obj['dtype'])
+    #    return globals(
+    #    )[obj['klass']](unconvert(obj['sp_values'], dtype, obj['compress']), sparse_index=obj['sp_index'],
+    #                    index=obj['index'], fill_value=obj['fill_value'], kind=obj['kind'], name=obj['name'])
+    #elif typ == 'sparse_dataframe':
+    #    return globals()[obj['klass']](obj['data'],
+    #                                   columns=obj['columns'], default_fill_value=obj['default_fill_value'], default_kind=obj['default_kind'])
+    #elif typ == 'sparse_panel':
+    #    return globals()[obj['klass']](obj['data'],
+    #                                   items=obj['items'], default_fill_value=obj['default_fill_value'], default_kind=obj['default_kind'])
     elif typ == 'block_index':
         return globals()[obj['klass']](obj['length'], obj['blocs'], obj['blengths'])
     elif typ == 'int_index':
@@ -460,7 +465,7 @@ def create_block(b):
 
 
 def pack(o, default=encode,
-         encoding='utf-8', unicode_errors='strict', use_single_float=False):
+         encoding='latin1', unicode_errors='strict', use_single_float=False):
     """
     Pack an object and return the packed bytes.
     """
@@ -471,7 +476,7 @@ def pack(o, default=encode,
 
 
 def unpack(packed, object_hook=decode,
-           list_hook=None, use_list=False, encoding='utf-8',
+           list_hook=None, use_list=False, encoding='latin1',
            unicode_errors='strict', object_pairs_hook=None):
     """
     Unpack a packed object, return an iterator
@@ -488,7 +493,7 @@ def unpack(packed, object_hook=decode,
 class Packer(_Packer):
 
     def __init__(self, default=encode,
-                 encoding='utf-8',
+                 encoding='latin1',
                  unicode_errors='strict',
                  use_single_float=False):
         super(Packer, self).__init__(default=default,
@@ -501,7 +506,7 @@ class Unpacker(_Unpacker):
 
     def __init__(self, file_like=None, read_size=0, use_list=False,
                  object_hook=decode,
-                 object_pairs_hook=None, list_hook=None, encoding='utf-8',
+                 object_pairs_hook=None, list_hook=None, encoding='latin1',
                  unicode_errors='strict', max_buffer_size=0):
         super(Unpacker, self).__init__(file_like=file_like,
                                        read_size=read_size,

pandas/io/tests/test_packers.py

@@ -61,30 +61,28 @@ class TestNumpy(Test):
     def test_numpy_scalar_float(self):
         x = np.float32(np.random.rand())
         x_rec = self.encode_decode(x)
-        self.assert_(np.allclose(x, x_rec) and type(x) == type(x_rec))
+        tm.assert_almost_equal(x,x_rec)
 
     def test_numpy_scalar_complex(self):
         x = np.complex64(np.random.rand() + 1j * np.random.rand())
         x_rec = self.encode_decode(x)
-        self.assert_(np.allclose(x, x_rec) and type(x) == type(x_rec))
+        tm.assert_almost_equal(x,x_rec)
 
     def test_scalar_float(self):
         x = np.random.rand()
         x_rec = self.encode_decode(x)
-        self.assert_(np.allclose(x, x_rec) and type(x) == type(x_rec))
+        tm.assert_almost_equal(x,x_rec)
 
     def test_scalar_complex(self):
         x = np.random.rand() + 1j * np.random.rand()
         x_rec = self.encode_decode(x)
-        self.assert_(np.allclose(x, x_rec) and type(x) == type(x_rec))
+        tm.assert_almost_equal(x,x_rec)
 
     def test_list_numpy_float(self):
         raise nose.SkipTest('buggy test')
         x = [np.float32(np.random.rand()) for i in range(5)]
         x_rec = self.encode_decode(x)
-        self.assert_(all(map(lambda x, y:
-                             x == y, x, x_rec)) and
-                     all(map(lambda x, y: type(x) == type(y), x, x_rec)))
+        tm.assert_almost_equal(x,x_rec)
 
     def test_list_numpy_float_complex(self):
         if not hasattr(np, 'complex128'):
@@ -96,65 +94,59 @@ def test_list_numpy_float_complex(self):
             [np.complex128(np.random.rand() + 1j * np.random.rand())
              for i in range(5)]
         x_rec = self.encode_decode(x)
-        self.assert_(all(map(lambda x, y: x == y, x, x_rec)) and
-                     all(map(lambda x, y: type(x) == type(y), x, x_rec)))
+        tm.assert_almost_equal(x,x_rec)
 
     def test_list_float(self):
         x = [np.random.rand() for i in range(5)]
         x_rec = self.encode_decode(x)
-        self.assert_(all(map(lambda x, y: x == y, x, x_rec)) and
-                     all(map(lambda x, y: type(x) == type(y), x, x_rec)))
+        tm.assert_almost_equal(x,x_rec)
 
     def test_list_float_complex(self):
         x = [np.random.rand() for i in range(5)] + \
             [(np.random.rand() + 1j * np.random.rand()) for i in range(5)]
         x_rec = self.encode_decode(x)
-        self.assert_(all(map(lambda x, y: x == y, x, x_rec)) and
-                     all(map(lambda x, y: type(x) == type(y), x, x_rec)))
+        tm.assert_almost_equal(x,x_rec)
 
     def test_dict_float(self):
         x = {'foo': 1.0, 'bar': 2.0}
         x_rec = self.encode_decode(x)
-        self.assert_(all(map(lambda x, y: x == y, x.values(), x_rec.values())) and
-                     all(map(lambda x, y: type(x) == type(y), x.values(), x_rec.values())))
+        tm.assert_almost_equal(x,x_rec)
 
     def test_dict_complex(self):
         x = {'foo': 1.0 + 1.0j, 'bar': 2.0 + 2.0j}
         x_rec = self.encode_decode(x)
-        self.assert_(all(map(lambda x, y: x == y, x.values(), x_rec.values())) and
-                     all(map(lambda x, y: type(x) == type(y), x.values(), x_rec.values())))
+        tm.assert_almost_equal(x,x_rec)
 
     def test_dict_numpy_float(self):
         x = {'foo': np.float32(1.0), 'bar': np.float32(2.0)}
         x_rec = self.encode_decode(x)
-        self.assert_(all(map(lambda x, y: x == y, x.values(), x_rec.values())) and
-                     all(map(lambda x, y: type(x) == type(y), x.values(), x_rec.values())))
+        tm.assert_almost_equal(x,x_rec)
 
     def test_dict_numpy_complex(self):
         x = {'foo': np.complex128(
             1.0 + 1.0j), 'bar': np.complex128(2.0 + 2.0j)}
         x_rec = self.encode_decode(x)
-        self.assert_(all(map(lambda x, y: x == y, x.values(), x_rec.values())) and
-                     all(map(lambda x, y: type(x) == type(y), x.values(), x_rec.values())))
+        tm.assert_almost_equal(x,x_rec)
 
     def test_numpy_array_float(self):
-        x = np.random.rand(5).astype(np.float32)
-        x_rec = self.encode_decode(x)
-        self.assert_(all(map(lambda x, y: x == y, x, x_rec)) and
-                     x.dtype == x_rec.dtype)
+
+        # run multiple times
+        for n in range(10):
+            x = np.random.rand(10)
+            for dtype in ['float32','float64']:
+                x = x.astype(dtype)
+                x_rec = self.encode_decode(x)
+                tm.assert_almost_equal(x,x_rec)
 
     def test_numpy_array_complex(self):
         x = (np.random.rand(5) + 1j * np.random.rand(5)).astype(np.complex128)
         x_rec = self.encode_decode(x)
-        self.assert_(all(map(lambda x, y: x == y, x, x_rec)) and
-                     x.dtype == x_rec.dtype)
+        tm.assert_almost_equal(x,x_rec)
 
     def test_list_mixed(self):
         x = [1.0, np.float32(3.5), np.complex128(4.25), u('foo')]
         x_rec = self.encode_decode(x)
-        self.assert_(all(map(lambda x, y: x == y, x, x_rec)) and
-                     all(map(lambda x, y: type(x) == type(y), x, x_rec)))
-
+        tm.assert_almost_equal(x,x_rec)
 
 class TestBasic(Test):
 
@@ -219,8 +211,12 @@ def test_multi_index(self):
 
     def test_unicode(self):
         i = tm.makeUnicodeIndex(100)
-        i_rec = self.encode_decode(i)
-        self.assert_(i.equals(i_rec))
+
+        # this currently fails
+        self.assertRaises(UnicodeEncodeError, self.encode_decode, i)
+
+        #i_rec = self.encode_decode(i)
+        #self.assert_(i.equals(i_rec))
 
 
 class TestSeries(Test):
@@ -255,9 +251,11 @@ def setUp(self):
 
     def test_basic(self):
 
-        for s, i in self.d.items():
-            i_rec = self.encode_decode(i)
-            assert_series_equal(i, i_rec)
+        # run multiple times here
+        for n in range(10):
+            for s, i in self.d.items():
+                i_rec = self.encode_decode(i)
+                assert_series_equal(i, i_rec)
 
 
 class TestNDFrame(Test):
@@ -326,8 +324,10 @@ class TestSparse(Test):
 
     def _check_roundtrip(self, obj, comparator, **kwargs):
 
-        i_rec = self.encode_decode(obj)
-        comparator(obj, i_rec, **kwargs)
+        # currently these are not implemetned
+        #i_rec = self.encode_decode(obj)
+        #comparator(obj, i_rec, **kwargs)
+        self.assertRaises(NotImplementedError, self.encode_decode, obj)
 
     def test_sparse_series(self):