Closed
Description
Zarr version
2.17.0
Numcodecs version
0.12.1
Python Version
3.12.1
Operating System
Fedora 39/40/Rawhide
Installation
pip into virtual environment; also from source
Description
When testing against fsspec 2024.2.0, some hierarchy tests fail. These tests passed with 2023.12.2 (the previous release).
Steps to reproduce
$ python -m venv venv
$ . venv/bin/activate
$ pip install pytest zarr
$ pip install fsspec==2023.12.2
$ pytest --pyargs zarr.tests.test_storage # All tests pass.
but then with fsspec 2024.2.0:
$ pip install fsspec==2024.2.0
$ pytest --pyargs zarr.tests.test_storage
================================================================================================================================ short test summary info =================================================================================================================================
FAILED venv/lib64/python3.12/site-packages/zarr/tests/test_storage.py::TestFSStore::test_hierarchy - AssertionError: assert [] == ['f']
FAILED venv/lib64/python3.12/site-packages/zarr/tests/test_storage.py::TestFSStoreWithKeySeparator::test_hierarchy - AssertionError: assert [] == ['f']
FAILED venv/lib64/python3.12/site-packages/zarr/tests/test_storage.py::TestFSStoreFromFilesystem::test_hierarchy - AssertionError: assert [] == ['f']
FAILED venv/lib64/python3.12/site-packages/zarr/tests/test_storage.py::TestN5FSStore::test_hierarchy - AssertionError: assert [] == ['f']
FAILED venv/lib64/python3.12/site-packages/zarr/tests/test_storage.py::TestNestedFSStore::test_hierarchy - AssertionError: assert [] == ['f']
================================================================================================================= 5 failed, 581 passed, 137 skipped, 1 warning in 1.67s ==================================================================================================================
Additional output
Full failure traceback output:
_______________________________________________________________________________________________________________________________ TestFSStore.test_hierarchy _______________________________________________________________________________________________________________________________
self = <zarr.tests.test_storage.TestFSStore object at 0x7f938a6bc8c0>
def test_hierarchy(self):
# setup
store = self.create_store()
store[self.root + "a"] = b"aaa"
store[self.root + "b"] = b"bbb"
store[self.root + "c/d"] = b"ddd"
store[self.root + "c/e/f"] = b"fff"
store[self.root + "c/e/g"] = b"ggg"
# check keys
assert self.root + "a" in store
assert self.root + "b" in store
assert self.root + "c/d" in store
assert self.root + "c/e/f" in store
assert self.root + "c/e/g" in store
assert self.root + "c" not in store
assert self.root + "c/" not in store
assert self.root + "c/e" not in store
assert self.root + "c/e/" not in store
assert self.root + "c/d/x" not in store
# check __getitem__
with pytest.raises(KeyError):
store[self.root + "c"]
with pytest.raises(KeyError):
store[self.root + "c/e"]
with pytest.raises(KeyError):
store[self.root + "c/d/x"]
# test getsize (optional)
if hasattr(store, "getsize"):
# TODO: proper behavior of getsize?
# v3 returns size of all nested arrays, not just the
# size of the arrays in the current folder.
if self.version == 2:
assert 6 == store.getsize()
else:
assert 15 == store.getsize()
assert 3 == store.getsize("a")
assert 3 == store.getsize("b")
if self.version == 2:
assert 3 == store.getsize("c")
else:
assert 9 == store.getsize("c")
assert 3 == store.getsize("c/d")
assert 6 == store.getsize("c/e")
assert 3 == store.getsize("c/e/f")
assert 3 == store.getsize("c/e/g")
# non-existent paths
assert 0 == store.getsize("x")
assert 0 == store.getsize("a/x")
assert 0 == store.getsize("c/x")
assert 0 == store.getsize("c/x/y")
assert 0 == store.getsize("c/d/y")
assert 0 == store.getsize("c/d/y/z")
# access item via full path
assert 3 == store.getsize(self.root + "a")
# test listdir (optional)
if hasattr(store, "listdir"):
assert {"a", "b", "c"} == set(store.listdir(self.root))
assert {"d", "e"} == set(store.listdir(self.root + "c"))
assert {"f", "g"} == set(store.listdir(self.root + "c/e"))
# no exception raised if path does not exist or is leaf
assert [] == store.listdir(self.root + "x")
assert [] == store.listdir(self.root + "a/x")
assert [] == store.listdir(self.root + "c/x")
assert [] == store.listdir(self.root + "c/x/y")
assert [] == store.listdir(self.root + "c/d/y")
assert [] == store.listdir(self.root + "c/d/y/z")
> assert [] == store.listdir(self.root + "c/e/f")
E AssertionError: assert [] == ['f']
E
E Right contains one more item: 'f'
E Use -v to get more diff
usr/local/lib/python3.12/site-packages/zarr/tests/test_storage.py:409: AssertionError
_______________________________________________________________________________________________________________________ TestFSStoreWithKeySeparator.test_hierarchy _______________________________________________________________________________________________________________________
self = <zarr.tests.test_storage.TestFSStoreWithKeySeparator object at 0x7f938a6bf170>
def test_hierarchy(self):
# setup
store = self.create_store()
store[self.root + "a"] = b"aaa"
store[self.root + "b"] = b"bbb"
store[self.root + "c/d"] = b"ddd"
store[self.root + "c/e/f"] = b"fff"
store[self.root + "c/e/g"] = b"ggg"
# check keys
assert self.root + "a" in store
assert self.root + "b" in store
assert self.root + "c/d" in store
assert self.root + "c/e/f" in store
assert self.root + "c/e/g" in store
assert self.root + "c" not in store
assert self.root + "c/" not in store
assert self.root + "c/e" not in store
assert self.root + "c/e/" not in store
assert self.root + "c/d/x" not in store
# check __getitem__
with pytest.raises(KeyError):
store[self.root + "c"]
with pytest.raises(KeyError):
store[self.root + "c/e"]
with pytest.raises(KeyError):
store[self.root + "c/d/x"]
# test getsize (optional)
if hasattr(store, "getsize"):
# TODO: proper behavior of getsize?
# v3 returns size of all nested arrays, not just the
# size of the arrays in the current folder.
if self.version == 2:
assert 6 == store.getsize()
else:
assert 15 == store.getsize()
assert 3 == store.getsize("a")
assert 3 == store.getsize("b")
if self.version == 2:
assert 3 == store.getsize("c")
else:
assert 9 == store.getsize("c")
assert 3 == store.getsize("c/d")
assert 6 == store.getsize("c/e")
assert 3 == store.getsize("c/e/f")
assert 3 == store.getsize("c/e/g")
# non-existent paths
assert 0 == store.getsize("x")
assert 0 == store.getsize("a/x")
assert 0 == store.getsize("c/x")
assert 0 == store.getsize("c/x/y")
assert 0 == store.getsize("c/d/y")
assert 0 == store.getsize("c/d/y/z")
# access item via full path
assert 3 == store.getsize(self.root + "a")
# test listdir (optional)
if hasattr(store, "listdir"):
assert {"a", "b", "c"} == set(store.listdir(self.root))
assert {"d", "e"} == set(store.listdir(self.root + "c"))
assert {"f", "g"} == set(store.listdir(self.root + "c/e"))
# no exception raised if path does not exist or is leaf
assert [] == store.listdir(self.root + "x")
assert [] == store.listdir(self.root + "a/x")
assert [] == store.listdir(self.root + "c/x")
assert [] == store.listdir(self.root + "c/x/y")
assert [] == store.listdir(self.root + "c/d/y")
assert [] == store.listdir(self.root + "c/d/y/z")
> assert [] == store.listdir(self.root + "c/e/f")
E AssertionError: assert [] == ['f']
E
E Right contains one more item: 'f'
E Use -v to get more diff
usr/local/lib/python3.12/site-packages/zarr/tests/test_storage.py:409: AssertionError
________________________________________________________________________________________________________________________ TestFSStoreFromFilesystem.test_hierarchy ________________________________________________________________________________________________________________________
self = <zarr.tests.test_storage.TestFSStoreFromFilesystem object at 0x7f938a6ec890>
def test_hierarchy(self):
# setup
store = self.create_store()
store[self.root + "a"] = b"aaa"
store[self.root + "b"] = b"bbb"
store[self.root + "c/d"] = b"ddd"
store[self.root + "c/e/f"] = b"fff"
store[self.root + "c/e/g"] = b"ggg"
# check keys
assert self.root + "a" in store
assert self.root + "b" in store
assert self.root + "c/d" in store
assert self.root + "c/e/f" in store
assert self.root + "c/e/g" in store
assert self.root + "c" not in store
assert self.root + "c/" not in store
assert self.root + "c/e" not in store
assert self.root + "c/e/" not in store
assert self.root + "c/d/x" not in store
# check __getitem__
with pytest.raises(KeyError):
store[self.root + "c"]
with pytest.raises(KeyError):
store[self.root + "c/e"]
with pytest.raises(KeyError):
store[self.root + "c/d/x"]
# test getsize (optional)
if hasattr(store, "getsize"):
# TODO: proper behavior of getsize?
# v3 returns size of all nested arrays, not just the
# size of the arrays in the current folder.
if self.version == 2:
assert 6 == store.getsize()
else:
assert 15 == store.getsize()
assert 3 == store.getsize("a")
assert 3 == store.getsize("b")
if self.version == 2:
assert 3 == store.getsize("c")
else:
assert 9 == store.getsize("c")
assert 3 == store.getsize("c/d")
assert 6 == store.getsize("c/e")
assert 3 == store.getsize("c/e/f")
assert 3 == store.getsize("c/e/g")
# non-existent paths
assert 0 == store.getsize("x")
assert 0 == store.getsize("a/x")
assert 0 == store.getsize("c/x")
assert 0 == store.getsize("c/x/y")
assert 0 == store.getsize("c/d/y")
assert 0 == store.getsize("c/d/y/z")
# access item via full path
assert 3 == store.getsize(self.root + "a")
# test listdir (optional)
if hasattr(store, "listdir"):
assert {"a", "b", "c"} == set(store.listdir(self.root))
assert {"d", "e"} == set(store.listdir(self.root + "c"))
assert {"f", "g"} == set(store.listdir(self.root + "c/e"))
# no exception raised if path does not exist or is leaf
assert [] == store.listdir(self.root + "x")
assert [] == store.listdir(self.root + "a/x")
assert [] == store.listdir(self.root + "c/x")
assert [] == store.listdir(self.root + "c/x/y")
assert [] == store.listdir(self.root + "c/d/y")
assert [] == store.listdir(self.root + "c/d/y/z")
> assert [] == store.listdir(self.root + "c/e/f")
E AssertionError: assert [] == ['f']
E
E Right contains one more item: 'f'
E Use -v to get more diff
usr/local/lib/python3.12/site-packages/zarr/tests/test_storage.py:409: AssertionError
______________________________________________________________________________________________________________________________ TestN5FSStore.test_hierarchy ______________________________________________________________________________________________________________________________
self = <zarr.tests.test_storage.TestN5FSStore object at 0x7f938a717680>
def test_hierarchy(self):
# setup
store = self.create_store()
store[self.root + "a"] = b"aaa"
store[self.root + "b"] = b"bbb"
store[self.root + "c/d"] = b"ddd"
store[self.root + "c/e/f"] = b"fff"
store[self.root + "c/e/g"] = b"ggg"
# check keys
assert self.root + "a" in store
assert self.root + "b" in store
assert self.root + "c/d" in store
assert self.root + "c/e/f" in store
assert self.root + "c/e/g" in store
assert self.root + "c" not in store
assert self.root + "c/" not in store
assert self.root + "c/e" not in store
assert self.root + "c/e/" not in store
assert self.root + "c/d/x" not in store
# check __getitem__
with pytest.raises(KeyError):
store[self.root + "c"]
with pytest.raises(KeyError):
store[self.root + "c/e"]
with pytest.raises(KeyError):
store[self.root + "c/d/x"]
# test getsize (optional)
if hasattr(store, "getsize"):
# TODO: proper behavior of getsize?
# v3 returns size of all nested arrays, not just the
# size of the arrays in the current folder.
if self.version == 2:
assert 6 == store.getsize()
else:
assert 15 == store.getsize()
assert 3 == store.getsize("a")
assert 3 == store.getsize("b")
if self.version == 2:
assert 3 == store.getsize("c")
else:
assert 9 == store.getsize("c")
assert 3 == store.getsize("c/d")
assert 6 == store.getsize("c/e")
assert 3 == store.getsize("c/e/f")
assert 3 == store.getsize("c/e/g")
# non-existent paths
assert 0 == store.getsize("x")
assert 0 == store.getsize("a/x")
assert 0 == store.getsize("c/x")
assert 0 == store.getsize("c/x/y")
assert 0 == store.getsize("c/d/y")
assert 0 == store.getsize("c/d/y/z")
# access item via full path
assert 3 == store.getsize(self.root + "a")
# test listdir (optional)
if hasattr(store, "listdir"):
assert {"a", "b", "c"} == set(store.listdir(self.root))
assert {"d", "e"} == set(store.listdir(self.root + "c"))
assert {"f", "g"} == set(store.listdir(self.root + "c/e"))
# no exception raised if path does not exist or is leaf
assert [] == store.listdir(self.root + "x")
assert [] == store.listdir(self.root + "a/x")
assert [] == store.listdir(self.root + "c/x")
assert [] == store.listdir(self.root + "c/x/y")
assert [] == store.listdir(self.root + "c/d/y")
assert [] == store.listdir(self.root + "c/d/y/z")
> assert [] == store.listdir(self.root + "c/e/f")
E AssertionError: assert [] == ['f']
E
E Right contains one more item: 'f'
E Use -v to get more diff
usr/local/lib/python3.12/site-packages/zarr/tests/test_storage.py:409: AssertionError
____________________________________________________________________________________________________________________________ TestNestedFSStore.test_hierarchy ____________________________________________________________________________________________________________________________
self = <zarr.tests.test_storage.TestNestedFSStore object at 0x7f938a7489b0>
def test_hierarchy(self):
# setup
store = self.create_store()
store[self.root + "a"] = b"aaa"
store[self.root + "b"] = b"bbb"
store[self.root + "c/d"] = b"ddd"
store[self.root + "c/e/f"] = b"fff"
store[self.root + "c/e/g"] = b"ggg"
# check keys
assert self.root + "a" in store
assert self.root + "b" in store
assert self.root + "c/d" in store
assert self.root + "c/e/f" in store
assert self.root + "c/e/g" in store
assert self.root + "c" not in store
assert self.root + "c/" not in store
assert self.root + "c/e" not in store
assert self.root + "c/e/" not in store
assert self.root + "c/d/x" not in store
# check __getitem__
with pytest.raises(KeyError):
store[self.root + "c"]
with pytest.raises(KeyError):
store[self.root + "c/e"]
with pytest.raises(KeyError):
store[self.root + "c/d/x"]
# test getsize (optional)
if hasattr(store, "getsize"):
# TODO: proper behavior of getsize?
# v3 returns size of all nested arrays, not just the
# size of the arrays in the current folder.
if self.version == 2:
assert 6 == store.getsize()
else:
assert 15 == store.getsize()
assert 3 == store.getsize("a")
assert 3 == store.getsize("b")
if self.version == 2:
assert 3 == store.getsize("c")
else:
assert 9 == store.getsize("c")
assert 3 == store.getsize("c/d")
assert 6 == store.getsize("c/e")
assert 3 == store.getsize("c/e/f")
assert 3 == store.getsize("c/e/g")
# non-existent paths
assert 0 == store.getsize("x")
assert 0 == store.getsize("a/x")
assert 0 == store.getsize("c/x")
assert 0 == store.getsize("c/x/y")
assert 0 == store.getsize("c/d/y")
assert 0 == store.getsize("c/d/y/z")
# access item via full path
assert 3 == store.getsize(self.root + "a")
# test listdir (optional)
if hasattr(store, "listdir"):
assert {"a", "b", "c"} == set(store.listdir(self.root))
assert {"d", "e"} == set(store.listdir(self.root + "c"))
assert {"f", "g"} == set(store.listdir(self.root + "c/e"))
# no exception raised if path does not exist or is leaf
assert [] == store.listdir(self.root + "x")
assert [] == store.listdir(self.root + "a/x")
assert [] == store.listdir(self.root + "c/x")
assert [] == store.listdir(self.root + "c/x/y")
assert [] == store.listdir(self.root + "c/d/y")
assert [] == store.listdir(self.root + "c/d/y/z")
> assert [] == store.listdir(self.root + "c/e/f")
E AssertionError: assert [] == ['f']
E
E Right contains one more item: 'f'
E Use -v to get more diff
usr/local/lib/python3.12/site-packages/zarr/tests/test_storage.py:409: AssertionError
==================================================================================================================================== warnings summary ====================================================================================================================================
usr/local/lib/python3.12/site-packages/zarr/tests/test_storage.py::TestZipStore::test_store_and_retrieve_ndarray
/usr/lib64/python3.12/zipfile/__init__.py:1590: UserWarning: Duplicate name: 'foo'
return self._open_to_write(zinfo, force_zip64=force_zip64)
-- Docs: https://docs.pytest.org/en/stable/how-to/capture-warnings.html
================================================================================================================================ short test summary info =================================================================================================================================
FAILED usr/local/lib/python3.12/site-packages/zarr/tests/test_storage.py::TestFSStore::test_hierarchy - AssertionError: assert [] == ['f']
FAILED usr/local/lib/python3.12/site-packages/zarr/tests/test_storage.py::TestFSStoreWithKeySeparator::test_hierarchy - AssertionError: assert [] == ['f']
FAILED usr/local/lib/python3.12/site-packages/zarr/tests/test_storage.py::TestFSStoreFromFilesystem::test_hierarchy - AssertionError: assert [] == ['f']
FAILED usr/local/lib/python3.12/site-packages/zarr/tests/test_storage.py::TestN5FSStore::test_hierarchy - AssertionError: assert [] == ['f']
FAILED usr/local/lib/python3.12/site-packages/zarr/tests/test_storage.py::TestNestedFSStore::test_hierarchy - AssertionError: assert [] == ['f']