Skip to content

RFC: Optionally support memory-mapping DirectoryStore values #377

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
wants to merge 9 commits into from
Next Next commit
Compare test data's content generally
Instead of strictly expecting values returned from a store to be `bytes`
instances only in the tests, relax this to anything that supports the
buffer protocol and has the expected content. This makes it possible to
return things like NumPy `memmap` objects and still have the tests work
correctly.
  • Loading branch information
jakirkham committed May 6, 2019
commit a860a320eee5f5b5c25f31fbde3e7799edd321f9
27 changes: 14 additions & 13 deletions zarr/tests/test_storage.py
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@
except ImportError: # pragma: no cover
asb = None

from numcodecs.compat import ensure_bytes

from zarr.storage import (init_array, array_meta_key, attrs_key, DictStore,
DirectoryStore, ZipStore, init_group, group_meta_key,
Expand Down Expand Up @@ -65,7 +66,7 @@ def test_get_set_del_contains(self):
store['foo']
store['foo'] = b'bar'
assert 'foo' in store
assert b'bar' == store['foo']
assert b'bar' == ensure_bytes(store['foo'])

# test __delitem__ (optional)
try:
Expand Down Expand Up @@ -103,10 +104,10 @@ def test_pop(self):
store['baz'] = b'qux'
assert len(store) == 2
v = store.pop('foo')
assert v == b'bar'
assert ensure_bytes(v) == b'bar'
assert len(store) == 1
v = store.pop('baz')
assert v == b'qux'
assert ensure_bytes(v) == b'qux'
assert len(store) == 0
with pytest.raises(KeyError):
store.pop('xxx')
Expand All @@ -122,7 +123,7 @@ def test_popitem(self):
store['foo'] = b'bar'
k, v = store.popitem()
assert k == 'foo'
assert v == b'bar'
assert ensure_bytes(v) == b'bar'
assert len(store) == 0
with pytest.raises(KeyError):
store.popitem()
Expand All @@ -141,8 +142,8 @@ def test_update(self):
assert 'foo' not in store
assert 'baz' not in store
store.update(foo=b'bar', baz=b'quux')
assert b'bar' == store['foo']
assert b'quux' == store['baz']
assert b'bar' == ensure_bytes(store['foo'])
assert b'quux' == ensure_bytes(store['baz'])

def test_iterators(self):
store = self.create_store()
Expand All @@ -164,9 +165,9 @@ def test_iterators(self):
assert 4 == len(store)
assert {'a', 'b', 'c/d', 'c/e/f'} == set(store)
assert {'a', 'b', 'c/d', 'c/e/f'} == set(store.keys())
assert {b'aaa', b'bbb', b'ddd', b'fff'} == set(store.values())
assert {b'aaa', b'bbb', b'ddd', b'fff'} == set(map(ensure_bytes, store.values()))
assert ({('a', b'aaa'), ('b', b'bbb'), ('c/d', b'ddd'), ('c/e/f', b'fff')} ==
set(store.items()))
set(map(lambda kv: (kv[0], ensure_bytes(kv[1])), store.items())))

def test_pickle(self):

Expand All @@ -190,8 +191,8 @@ def test_pickle(self):
# verify
assert n == len(store2)
assert keys == sorted(store2.keys())
assert b'bar' == store2['foo']
assert b'quux' == store2['baz']
assert b'bar' == ensure_bytes(store2['foo'])
assert b'quux' == ensure_bytes(store2['baz'])

def test_getsize(self):
store = self.create_store()
Expand Down Expand Up @@ -671,10 +672,10 @@ def setdel_hierarchy_checks(store):
# test __setitem__ overwrite level
store['x/y/z'] = b'xxx'
store['x/y'] = b'yyy'
assert b'yyy' == store['x/y']
assert b'yyy' == ensure_bytes(store['x/y'])
assert 'x/y/z' not in store
store['x'] = b'zzz'
assert b'zzz' == store['x']
assert b'zzz' == ensure_bytes(store['x'])
assert 'x/y' not in store

# test __delitem__ overwrite level
Expand Down Expand Up @@ -736,7 +737,7 @@ def test_pickle_ext(self):
# check point to same underlying directory
assert 'xxx' not in store
store2['xxx'] = b'yyy'
assert b'yyy' == store['xxx']
assert b'yyy' == ensure_bytes(store['xxx'])

def test_setdel(self):
store = self.create_store()
Expand Down