Introduce a flat option to ensure_contiguous_ndarray to switch off flatten for ZFPY codec (#307)

* Optional flatten the encode array

* Remove skipping Python 3.9 and later for Zfp

* Updated docs/release.rst about ZFPY

* Removed the trailing whitespace

* Add pip install zfpy to start test_zfpy

* Add raise valueerror

* Shorten the raise error lines

* Modified ci-osx clang to 12.0.1

* Revert the change in ci-osx.yaml

* Update release.rst

* Change flat to flatten

* Missed one flat, changed to flatten

* Change flat to flatten in zfpy.py

Author: halehawk

docs/conf.py

@@ -24,7 +24,7 @@ def __getattr__(cls, name):
         return Mock()
 
 
-MOCK_MODULES = ['msgpack', 'zfpy']
+MOCK_MODULES = ['msgpack']
 sys.modules.update((mod_name, Mock()) for mod_name in MOCK_MODULES)
 
 

docs/release.rst

@@ -5,6 +5,10 @@ Release notes
 
 Unreleased
 ----------
+* Fix a flatten array error for ZFPY, ZFPY codec is supported on Python 3.9
+  and 3.10 on Linux and MacOS, the docs about ZFPY is also available.
+  By :user:`Haiying Xu <halehawk>`, `John Kirkham <jakirkham>`, `Ryan Abernathey <rabernat>` :
+  issue:`303`.
 
 .. _release_0.9.1:
 

numcodecs/compat.py

@@ -50,7 +50,7 @@ def ensure_ndarray(buf):
     return arr
 
 
-def ensure_contiguous_ndarray(buf, max_buffer_size=None):
+def ensure_contiguous_ndarray(buf, max_buffer_size=None, flatten=True):
     """Convenience function to coerce `buf` to a numpy array, if it is not already a
     numpy array. Also ensures that the returned value exports fully contiguous memory,
     and supports the new-style buffer interface. If the optional max_buffer_size is
@@ -91,8 +91,10 @@ def ensure_contiguous_ndarray(buf, max_buffer_size=None):
 
     # check memory is contiguous, if so flatten
     if arr.flags.c_contiguous or arr.flags.f_contiguous:
-        # can flatten without copy
-        arr = arr.reshape(-1, order='A')
+        # check if flatten flag is on or not
+        if flatten:
+            # can flatten without copy
+            arr = arr.reshape(-1, order='A')
 
     else:
         raise ValueError('an array with contiguous memory is required')

numcodecs/tests/test_zfpy.py

@@ -38,7 +38,6 @@
     np.random.normal(loc=1000, scale=1, size=(100, 10)),
     np.random.normal(loc=1000, scale=1, size=(10, 10, 10)),
     np.random.normal(loc=1000, scale=1, size=(2, 5, 10, 10)),
-    np.asfortranarray(np.random.normal(loc=1000, scale=1, size=(5, 10, 20))),
     np.random.randint(-(2 ** 31), -(2 ** 31) + 20, size=1000, dtype="i4").reshape(
         100, 10
     ),
@@ -84,3 +83,26 @@ def test_err_decode_object_buffer():
 
 def test_err_encode_object_buffer():
     check_err_encode_object_buffer(ZFPY())
+
+
+def test_err_encode_list():
+    data = ['foo', 'bar', 'baz']
+    for codec in codecs:
+        with pytest.raises(TypeError):
+            codec.encode(data)
+
+
+def test_err_encode_non_contiguous():
+    # non-contiguous memory
+    arr = np.arange(1000, dtype='i4')[::2]
+    for codec in codecs:
+        with pytest.raises(ValueError):
+            codec.encode(arr)
+
+
+def test_err_encode_fortran_array():
+    # fortran array
+    arr = np.asfortranarray(np.random.normal(loc=1000, scale=1, size=(5, 10, 20)))
+    for codec in codecs:
+        with pytest.raises(ValueError):
+            codec.encode(arr)

numcodecs/zfpy.py

@@ -9,6 +9,7 @@
 
     from .abc import Codec
     from .compat import ndarray_copy, ensure_contiguous_ndarray, ensure_bytes
+    import numpy as np
 
     # noinspection PyShadowingBuiltins
     class ZFPY(Codec):
@@ -54,8 +55,16 @@ def __init__(
 
         def encode(self, buf):
 
-            # normalise inputs
-            buf = ensure_contiguous_ndarray(buf)
+            # not flatten c-order array and raise exception for f-order array
+            if not isinstance(buf, np.ndarray):
+                raise TypeError("The zfp codec does not support none numpy arrays."
+                                f" Your buffers were {type(buf)}.")
+            if buf.flags.c_contiguous:
+                flatten = False
+            else:
+                raise ValueError("The zfp codec does not support F order arrays. "
+                                 f"Your arrays flags were {buf.flags}.")
+            buf = ensure_contiguous_ndarray(buf, flatten=flatten)
 
             # do compression
             return _zfpy.compress_numpy(

requirements.txt

@@ -2,3 +2,4 @@ cython
 numpy
 msgpack
 pytest
+zfpy

requirements_dev.txt

@@ -1,5 +1,6 @@
 Cython==0.29.21
 msgpack==1.0.2
 numpy==1.21.0
-zfpy==0.5.5; python_version < '3.9'
+zfpy==0.5.5
+
 

requirements_rtfd.txt

@@ -6,4 +6,4 @@ numpydoc
 mock
 numpy
 cython
-zfpy==0.5.5; python_version < '3.9'
+zfpy==0.5.5