Refactor and use more buffer view coercion

numcodecs/bz2.py

@@ -1,14 +1,13 @@
 # -*- coding: utf-8 -*-
 from __future__ import absolute_import, print_function, division
 import bz2 as _bz2
-import array
 
 
 import numpy as np
 
 
 from numcodecs.abc import Codec
-from numcodecs.compat import buffer_copy, handle_datetime
+from numcodecs.compat import buffer_copy, to_buffer
 
 
 class BZ2(Codec):
@@ -28,29 +27,21 @@ def __init__(self, level=1):
 
     def encode(self, buf):
 
-        # deal with lack of buffer support for datetime64 and timedelta64
-        buf = handle_datetime(buf)
-
         if isinstance(buf, np.ndarray):
 
             # cannot compress object array
             if buf.dtype == object:
                 raise ValueError('cannot encode object array')
 
-            # if numpy array, can only handle C contiguous directly
-            if not buf.flags.c_contiguous:
-                buf = buf.tobytes(order='A')
+        buf = to_buffer(buf)
 
         # do compression
         return _bz2.compress(buf, self.level)
 
     # noinspection PyMethodMayBeStatic
     def decode(self, buf, out=None):
 
-        # BZ2 cannot handle ndarray directly at all, coerce everything to
-        # memoryview
-        if not isinstance(buf, array.array):
-            buf = memoryview(buf)
+        buf = to_buffer(buf)
 
         # do decompression
         dec = _bz2.decompress(buf)

numcodecs/compat.py

@@ -14,29 +14,47 @@
 
 if PY2:  # pragma: py3 no cover
 
+    buffer = buffer
     text_type = unicode
     binary_type = str
     integer_types = (int, long)
     reduce = reduce
 
 else:  # pragma: py2 no cover
 
+    buffer = memoryview
     text_type = str
     binary_type = bytes
     integer_types = int,
     from functools import reduce
 
 
+def to_buffer(v):
+    """Obtain a `buffer` or `memoryview` for `v`."""
+    b = v
+    if isinstance(b, np.ndarray):
+        if b.dtype.kind is 'O':
+            raise ValueError('cannot encode object array')
+        b = b.reshape(-1, order='A')
+        if b.dtype.kind in 'Mm':
+            b = b.view(np.uint8)
+        b = buffer(b)
+    elif PY2:  # pragma: py3 no cover
+        if not isinstance(b, array.array) and memoryview(b).format is 'O':
+            raise ValueError('cannot encode object array')
+        b = buffer(b)
+    else:      # pragma: py2 no cover
+        b = buffer(b)
+        if b.format is 'O':
+            raise ValueError('cannot encode object array')
+        b = b.cast('B').cast(b.format)
+
+    return b
+
+
 def buffer_tobytes(v):
     """Obtain a sequence of bytes for the memory buffer used by `v`."""
-    if isinstance(v, binary_type):
-        return v
-    elif isinstance(v, np.ndarray):
-        return v.tobytes(order='A')
-    elif PY2 and isinstance(v, array.array):  # pragma: py3 no cover
-        return v.tostring()
-    else:
-        return memoryview(v).tobytes()
+    return memoryview(to_buffer(v)).tobytes()
 
 
 def buffer_copy(buf, out=None):
@@ -98,9 +116,3 @@ def ensure_text(l, encoding='utf-8'):
         return l
     else:  # pragma: py3 no cover
         return text_type(l, encoding=encoding)
-
-
-def handle_datetime(buf):
-    if hasattr(buf, 'dtype') and buf.dtype.kind in 'Mm':
-        return buf.view('u8')
-    return buf

numcodecs/gzip.py

@@ -8,7 +8,7 @@
 
 
 from .abc import Codec
-from .compat import buffer_copy, handle_datetime, buffer_tobytes, PY2
+from .compat import buffer_copy, to_buffer
 
 
 class GZip(Codec):
@@ -28,22 +28,13 @@ def __init__(self, level=1):
 
     def encode(self, buf):
 
-        # deal with lack of buffer support for datetime64 and timedelta64
-        buf = handle_datetime(buf)
-
         if isinstance(buf, np.ndarray):
 
             # cannot compress object array
             if buf.dtype == object:
                 raise ValueError('cannot encode object array')
 
-            # if numpy array, can only handle C contiguous directly
-            if not buf.flags.c_contiguous:
-                buf = buf.tobytes(order='A')
-
-        if PY2:  # pragma: py3 no cover
-            # ensure bytes, PY2 cannot handle things like bytearray
-            buf = buffer_tobytes(buf)
+        buf = to_buffer(buf)
 
         # do compression
         compressed = io.BytesIO()
@@ -58,9 +49,7 @@ def encode(self, buf):
     # noinspection PyMethodMayBeStatic
     def decode(self, buf, out=None):
 
-        if PY2:  # pragma: py3 no cover
-            # ensure bytes, PY2 cannot handle things like bytearray
-            buf = buffer_tobytes(buf)
+        buf = to_buffer(buf)
 
         # do decompression
         buf = io.BytesIO(buf)

numcodecs/lzma.py

@@ -16,7 +16,7 @@
 
     import numpy as np
     from .abc import Codec
-    from .compat import buffer_copy, handle_datetime
+    from .compat import buffer_copy, to_buffer
 
     # noinspection PyShadowingBuiltins
     class LZMA(Codec):
@@ -48,25 +48,22 @@ def __init__(self, format=1, check=-1, preset=None, filters=None):
 
         def encode(self, buf):
 
-            # deal with lack of buffer support for datetime64 and timedelta64
-            buf = handle_datetime(buf)
-
             if isinstance(buf, np.ndarray):
 
                 # cannot compress object array
                 if buf.dtype == object:
                     raise ValueError('cannot encode object array')
 
-                # if numpy array, can only handle C contiguous directly
-                if not buf.flags.c_contiguous:
-                    buf = buf.tobytes(order='A')
+            buf = to_buffer(buf)
 
             # do compression
             return _lzma.compress(buf, format=self.format, check=self.check,
                                   preset=self.preset, filters=self.filters)
 
         def decode(self, buf, out=None):
 
+            buf = to_buffer(buf)
+
             # do decompression
             dec = _lzma.decompress(buf, format=self.format,
                                    filters=self.filters)

numcodecs/tests/common.py

@@ -187,7 +187,8 @@ def check_backwards_compatibility(codec_id, arrays, codecs, precision=None, pref
         # setup
         i = int(arr_fn.split('.')[-2])
         arr = np.load(arr_fn)
-        arr_bytes = buffer_tobytes(arr)
+        if arr.dtype.kind is not 'O':
+            arr_bytes = buffer_tobytes(arr)
         if arr.flags.f_contiguous:
             order = 'F'
         else:

numcodecs/tests/test_compat.py

@@ -2,13 +2,22 @@
 from __future__ import absolute_import, print_function, division
 import array
 
+import pytest
 
 import numpy as np
 
 
 from numcodecs.compat import buffer_tobytes
 
 
+def test_buffer_tobytes_raises():
+    a = np.array([u'Xin chào thế giới'], dtype=object)
+    with pytest.raises(ValueError):
+        buffer_tobytes(a)
+    with pytest.raises(ValueError):
+        buffer_tobytes(memoryview(a))
+
+
 def test_buffer_tobytes():
     bufs = [
         b'adsdasdas',

numcodecs/zlib.py

@@ -7,7 +7,7 @@
 
 
 from .abc import Codec
-from .compat import buffer_copy, handle_datetime, buffer_tobytes, PY2
+from .compat import buffer_copy, to_buffer
 
 
 class Zlib(Codec):
@@ -27,32 +27,21 @@ def __init__(self, level=1):
 
     def encode(self, buf):
 
-        # deal with lack of buffer support for datetime64 and timedelta64
-        buf = handle_datetime(buf)
-
         if isinstance(buf, np.ndarray):
 
             # cannot compress object array
             if buf.dtype == object:
                 raise ValueError('cannot encode object array')
 
-            # if numpy array, can only handle C contiguous directly
-            if not buf.flags.c_contiguous:
-                buf = buf.tobytes(order='A')
-
-        if PY2:  # pragma: py3 no cover
-            # ensure bytes, PY2 cannot handle things like bytearray
-            buf = buffer_tobytes(buf)
+        buf = to_buffer(buf)
 
         # do compression
         return _zlib.compress(buf, self.level)
 
     # noinspection PyMethodMayBeStatic
     def decode(self, buf, out=None):
 
-        if PY2:  # pragma: py3 no cover
-            # ensure bytes, PY2 cannot handle things like bytearray
-            buf = buffer_tobytes(buf)
+        buf = to_buffer(buf)
 
         # do decompression
         dec = _zlib.decompress(buf)