Refactor and use more buffer view coercion

numcodecs/astype.py

@@ -50,7 +50,8 @@ def __init__(self, encode_dtype, decode_dtype):
     def encode(self, buf):
 
         # view input data as 1D array
-        arr = ndarray_from_buffer(buf, self.decode_dtype)
+        arr = ndarray_from_buffer(buf)
+        arr = arr.view(self.decode_dtype).reshape(-1, order='A')
 
         # convert and copy
         enc = arr.astype(self.encode_dtype)
@@ -60,7 +61,8 @@ def encode(self, buf):
     def decode(self, buf, out=None):
 
         # view encoded data as 1D array
-        enc = ndarray_from_buffer(buf, self.encode_dtype)
+        enc = ndarray_from_buffer(buf)
+        enc = enc.view(self.encode_dtype).reshape(-1, order='A')
 
         # convert and copy
         dec = enc.astype(self.decode_dtype)

numcodecs/bz2.py

@@ -1,14 +1,10 @@
 # -*- coding: utf-8 -*-
 from __future__ import absolute_import, print_function, division
 import bz2 as _bz2
-import array
-
-
-import numpy as np
 
 
 from numcodecs.abc import Codec
-from numcodecs.compat import buffer_copy, handle_datetime
+from numcodecs.compat import buffer_copy, to_buffer
 
 
 class BZ2(Codec):
@@ -28,29 +24,15 @@ def __init__(self, level=1):
 
     def encode(self, buf):
 
-        # deal with lack of buffer support for datetime64 and timedelta64
-        buf = handle_datetime(buf)
-
-        if isinstance(buf, np.ndarray):
-
-            # cannot compress object array
-            if buf.dtype == object:
-                raise ValueError('cannot encode object array')
-
-            # if numpy array, can only handle C contiguous directly
-            if not buf.flags.c_contiguous:
-                buf = buf.tobytes(order='A')
+        buf = to_buffer(buf)
 
         # do compression
         return _bz2.compress(buf, self.level)
 
     # noinspection PyMethodMayBeStatic
     def decode(self, buf, out=None):
 
-        # BZ2 cannot handle ndarray directly at all, coerce everything to
-        # memoryview
-        if not isinstance(buf, array.array):
-            buf = memoryview(buf)
+        buf = memoryview(to_buffer(buf))
 
         # do decompression
         dec = _bz2.decompress(buf)

numcodecs/categorize.py

@@ -53,7 +53,8 @@ def __init__(self, labels, dtype, astype='u1'):
     def encode(self, buf):
 
         # view input as ndarray
-        arr = ndarray_from_buffer(buf, self.dtype)
+        arr = ndarray_from_buffer(buf)
+        arr = arr.view(self.dtype).reshape(-1, order='A')
 
         # setup output array
         enc = np.zeros_like(arr, dtype=self.astype)
@@ -67,7 +68,8 @@ def encode(self, buf):
     def decode(self, buf, out=None):
 
         # view encoded data as ndarray
-        enc = ndarray_from_buffer(buf, self.astype)
+        enc = ndarray_from_buffer(buf)
+        enc = enc.view(self.astype).reshape(-1, order='A')
 
         # setup output
         if isinstance(out, np.ndarray):

numcodecs/checksum32.py

@@ -17,15 +17,17 @@ class Checksum32(Codec):
     def encode(self, buf):
         if isinstance(buf, np.ndarray) and buf.dtype == object:
             raise ValueError('cannot encode object array')
-        arr = ndarray_from_buffer(buf, dtype='u1')
+        arr = ndarray_from_buffer(buf)
+        arr = arr.view('u1').reshape(-1, order='A')
         checksum = self.checksum(arr) & 0xffffffff
         enc = np.empty(arr.nbytes + 4, dtype='u1')
         enc[:4].view('<u4')[0] = checksum
         enc[4:] = arr
         return enc
 
     def decode(self, buf, out=None):
-        arr = ndarray_from_buffer(buf, dtype='u1')
+        arr = ndarray_from_buffer(buf)
+        arr = arr.view('u1').reshape(-1, order='A')
         expect = arr[:4].view('<u4')[0]
         checksum = self.checksum(arr[4:]) & 0xffffffff
         if expect != checksum:

numcodecs/compat.py

@@ -27,16 +27,19 @@
     from functools import reduce
 
 
-def buffer_tobytes(v):
-    """Obtain a sequence of bytes for the memory buffer used by `v`."""
-    if isinstance(v, binary_type):
-        return v
-    elif isinstance(v, np.ndarray):
-        return v.tobytes(order='A')
-    elif PY2 and isinstance(v, array.array):  # pragma: py3 no cover
-        return v.tostring()
-    else:
-        return memoryview(v).tobytes()
+def to_buffer(v):
+    """Obtain a `buffer` or `memoryview` for `v`."""
+
+    b = ndarray_from_buffer(v)
+
+    if b.dtype.kind is 'O':
+        raise ValueError('cannot encode object array')
+    elif b.dtype.kind in 'Mm':
+        b = b.view(np.int64)
+
+    b = b.reshape(-1, order='A')
+
+    return b
 
 
 def buffer_copy(buf, out=None):
@@ -46,50 +49,50 @@ def buffer_copy(buf, out=None):
         # no-op
         return buf
 
-    # handle ndarray destination
-    if isinstance(out, np.ndarray):
-
-        # view source as destination dtype
-        if isinstance(buf, np.ndarray):
-            buf = buf.view(dtype=out.dtype).reshape(-1, order='A')
-        else:
-            buf = np.frombuffer(buf, dtype=out.dtype)
-
-        # ensure shapes are compatible
-        if buf.shape != out.shape:
-            if out.flags.f_contiguous:
-                order = 'F'
-            else:
-                order = 'C'
-            buf = buf.reshape(out.shape, order=order)
-
-        # copy via numpy
-        np.copyto(out, buf)
-
-    # handle generic buffer destination
-    else:
+    # coerce to ndarrays
+    buf = ndarray_from_buffer(buf)
+    out = ndarray_from_buffer(out)
 
-        # obtain memoryview of destination
-        dest = memoryview(out)
+    # view source as destination dtype
+    if out.dtype.kind is not 'O':
+        buf = buf.view(out.dtype)
 
-        # ensure source is 1D
-        if isinstance(buf, np.ndarray):
-            buf = buf.reshape(-1, order='A')
-            # try to match itemsize
-            dtype = 'u%s' % dest.itemsize
-            buf = buf.view(dtype=dtype)
+    # ensure shapes are compatible
+    buf = buf.reshape(-1, order='A')
+    if buf.shape != out.shape:
+        if out.flags.f_contiguous:
+            order = 'F'
+        else:
+            order = 'C'
+        buf = buf.reshape(out.shape, order=order)
 
-        # try to copy via memoryview
-        dest[:] = buf
+    # copy via numpy
+    np.copyto(out, buf)
 
     return out
 
 
-def ndarray_from_buffer(buf, dtype):
-    if isinstance(buf, np.ndarray):
-        arr = buf.reshape(-1, order='A').view(dtype)
-    else:
-        arr = np.frombuffer(buf, dtype=dtype)
+def ndarray_from_buffer(buf):
+    arr = buf
+    if not isinstance(arr, np.ndarray):
+        try:
+            arr = memoryview(arr)
+        except TypeError:
+            if PY2:  # pragma: py3 no cover
+                arr = np.getbuffer(arr)
+            else:
+                raise
+        else:  # pragma: py2 no cover
+            if isinstance(buf, array.array) and buf.typecode is 'u':
+                arr = arr.cast('B').cast(np.dtype('u%i' % buf.itemsize).char)
+
+        arr = np.array(arr, copy=False)
+        if PY2 and isinstance(buf, array.array):  # pragma: py3 no cover
+            if buf.typecode is 'u':
+                arr = arr.view('u%i' % buf.itemsize)
+            else:
+                arr = arr.view(buf.typecode)
+
     return arr
 
 
@@ -98,9 +101,3 @@ def ensure_text(l, encoding='utf-8'):
         return l
     else:  # pragma: py3 no cover
         return text_type(l, encoding=encoding)
-
-
-def handle_datetime(buf):
-    if hasattr(buf, 'dtype') and buf.dtype.kind in 'Mm':
-        return buf.view('i8')
-    return buf

numcodecs/delta.py

@@ -57,7 +57,8 @@ def __init__(self, dtype, astype=None):
     def encode(self, buf):
 
         # view input data as 1D array
-        arr = ndarray_from_buffer(buf, self.dtype)
+        arr = ndarray_from_buffer(buf)
+        arr = arr.view(self.dtype).reshape(-1, order='A')
 
         # setup encoded output
         enc = np.empty_like(arr, dtype=self.astype)
@@ -73,7 +74,8 @@ def encode(self, buf):
     def decode(self, buf, out=None):
 
         # view encoded data as 1D array
-        enc = ndarray_from_buffer(buf, self.astype)
+        enc = ndarray_from_buffer(buf)
+        enc = enc.view(self.astype).reshape(-1, order='A')
 
         # setup decoded output
         if isinstance(out, np.ndarray):

numcodecs/fixedscaleoffset.py

@@ -88,7 +88,8 @@ def __init__(self, offset, scale, dtype, astype=None):
     def encode(self, buf):
 
         # interpret buffer as 1D array
-        arr = ndarray_from_buffer(buf, self.dtype)
+        arr = ndarray_from_buffer(buf)
+        arr = arr.view(self.dtype).reshape(-1, order='A')
 
         # compute scale offset
         enc = (arr - self.offset) * self.scale
@@ -104,7 +105,8 @@ def encode(self, buf):
     def decode(self, buf, out=None):
 
         # interpret buffer as 1D array
-        enc = ndarray_from_buffer(buf, self.astype)
+        enc = ndarray_from_buffer(buf)
+        enc = enc.view(self.astype).reshape(-1, order='A')
 
         # decode scale offset
         dec = (enc / self.scale) + self.offset

numcodecs/gzip.py

@@ -4,11 +4,8 @@
 import io
 
 
-import numpy as np
-
-
 from .abc import Codec
-from .compat import buffer_copy, handle_datetime, buffer_tobytes, PY2
+from .compat import buffer_copy, to_buffer
 
 
 class GZip(Codec):
@@ -28,22 +25,7 @@ def __init__(self, level=1):
 
     def encode(self, buf):
 
-        # deal with lack of buffer support for datetime64 and timedelta64
-        buf = handle_datetime(buf)
-
-        if isinstance(buf, np.ndarray):
-
-            # cannot compress object array
-            if buf.dtype == object:
-                raise ValueError('cannot encode object array')
-
-            # if numpy array, can only handle C contiguous directly
-            if not buf.flags.c_contiguous:
-                buf = buf.tobytes(order='A')
-
-        if PY2:  # pragma: py3 no cover
-            # ensure bytes, PY2 cannot handle things like bytearray
-            buf = buffer_tobytes(buf)
+        buf = memoryview(to_buffer(buf))
 
         # do compression
         compressed = io.BytesIO()
@@ -58,9 +40,7 @@ def encode(self, buf):
     # noinspection PyMethodMayBeStatic
     def decode(self, buf, out=None):
 
-        if PY2:  # pragma: py3 no cover
-            # ensure bytes, PY2 cannot handle things like bytearray
-            buf = buffer_tobytes(buf)
+        buf = to_buffer(buf)
 
         # do decompression
         buf = io.BytesIO(buf)

numcodecs/json.py

@@ -8,7 +8,7 @@
 
 
 from .abc import Codec
-from .compat import buffer_tobytes
+from .compat import to_buffer
 
 
 class JSON(Codec):
@@ -63,7 +63,7 @@ def encode(self, buf):
         return self._encoder.encode(items).encode(self._text_encoding)
 
     def decode(self, buf, out=None):
-        buf = buffer_tobytes(buf)
+        buf = to_buffer(buf).tobytes()
         items = self._decoder.decode(buf.decode(self._text_encoding))
         dec = np.empty(items[-1], dtype=items[-2])
         dec[:] = items[:-2]
@@ -125,7 +125,7 @@ def encode(self, buf):
         return self._encoder.encode(items).encode(self._text_encoding)
 
     def decode(self, buf, out=None):
-        buf = buffer_tobytes(buf)
+        buf = to_buffer(buf).tobytes()
         items = self._decoder.decode(buf.decode(self._text_encoding))
         dec = np.array(items[:-1], dtype=items[-1])
         if out is not None:

numcodecs/lzma.py

@@ -14,9 +14,8 @@
 
 if _lzma:
 
-    import numpy as np
     from .abc import Codec
-    from .compat import buffer_copy, handle_datetime
+    from .compat import buffer_copy, to_buffer
 
     # noinspection PyShadowingBuiltins
     class LZMA(Codec):
@@ -48,25 +47,16 @@ def __init__(self, format=1, check=-1, preset=None, filters=None):
 
         def encode(self, buf):
 
-            # deal with lack of buffer support for datetime64 and timedelta64
-            buf = handle_datetime(buf)
-
-            if isinstance(buf, np.ndarray):
-
-                # cannot compress object array
-                if buf.dtype == object:
-                    raise ValueError('cannot encode object array')
-
-                # if numpy array, can only handle C contiguous directly
-                if not buf.flags.c_contiguous:
-                    buf = buf.tobytes(order='A')
+            buf = to_buffer(buf)
 
             # do compression
             return _lzma.compress(buf, format=self.format, check=self.check,
                                   preset=self.preset, filters=self.filters)
 
         def decode(self, buf, out=None):
 
+            buf = to_buffer(buf)
+
             # do decompression
             dec = _lzma.decompress(buf, format=self.format,
                                    filters=self.filters)