Skip to content

make shardingcodec pickleable #2011

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 12 commits into from
Jul 5, 2024
Merged

make shardingcodec pickleable #2011

Changes from 1 commit
Commits
Show all changes
12 commits
Select commit Hold shift + click to select a range
6ad2ca6
use tmpdir for test
d-v-b Jun 23, 2024
2ffffd8
Merge branch 'v3' of https://github.com/zarr-developers/zarr-python i…
d-v-b Jul 1, 2024
12c98dd
Merge branch 'v3' of https://github.com/zarr-developers/zarr-python i…
d-v-b Jul 3, 2024
e29efd1
type annotations
d-v-b Jul 3, 2024
c4e49a6
refactor morton decode and remove destructuring in call to max
d-v-b Jul 3, 2024
34eed0a
parametrize sharding codec test by data shape
d-v-b Jul 3, 2024
0a36e2f
refactor codec tests
d-v-b Jul 3, 2024
0d40ce5
add test for pickling sharding codec, and make it pass
d-v-b Jul 3, 2024
8b6e8ee
Revert "use tmpdir for test"
d-v-b Jul 3, 2024
6c1035e
move fixtures into conftest.py
d-v-b Jul 3, 2024
dad3c5f
Update tests/v3/test_codecs/test_endian.py
d-v-b Jul 5, 2024
c334ba3
Merge branch 'v3' into pickle_tests
d-v-b Jul 5, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Next Next commit
use tmpdir for test
  • Loading branch information
@d-v-b
d-v-b committed Jun 23, 2024
commit 6ad2ca61b78fd1ecfc10d7fc80ae5055ed1a9d8b
61 changes: 30 additions & 31 deletions tests/v2/test_storage.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2227,38 +2227,38 @@ def test_format_compatibility():
np.random.seed(42)

arrays_chunks = [
(np.arange(1111, dtype="<i1"), 100),
(np.arange(1111, dtype="<i2"), 100),
(np.arange(1111, dtype="<i4"), 100),
(np.arange(1111, dtype="<i8"), 1000),
(np.arange(1111, dtype="<i1"), 600),
(np.arange(1111, dtype="<i2"), 600),
(np.arange(1111, dtype="<i4"), 600),
(np.arange(1111, dtype="<i8"), 600),
(np.random.randint(0, 200, size=2222, dtype="u1").astype("<u1"), 100),
(np.random.randint(0, 2000, size=2222, dtype="u2").astype("<u2"), 100),
(np.random.randint(0, 2000, size=2222, dtype="u4").astype("<u4"), 100),
(np.random.randint(0, 2000, size=2222, dtype="u8").astype("<u8"), 100),
(np.linspace(0, 1, 3333, dtype="<f2"), 100),
(np.linspace(0, 1, 3333, dtype="<f4"), 100),
(np.linspace(0, 1, 3333, dtype="<f8"), 100),
(np.random.normal(loc=0, scale=1, size=4444).astype("<f2"), 100),
(np.random.normal(loc=0, scale=1, size=4444).astype("<f4"), 100),
(np.random.normal(loc=0, scale=1, size=4444).astype("<f8"), 100),
(np.random.choice([b"A", b"C", b"G", b"T"], size=5555, replace=True).astype("S"), 100),
(np.random.randint(0, 2000, size=2222, dtype="u2").astype("<u2"), 1000),
(np.random.randint(0, 2000, size=2222, dtype="u4").astype("<u4"), 1000),
(np.random.randint(0, 2000, size=2222, dtype="u8").astype("<u8"), 1000),
(np.linspace(0, 1, 3333, dtype="<f2"), 600),
(np.linspace(0, 1, 3333, dtype="<f4"), 600),
(np.linspace(0, 1, 3333, dtype="<f8"), 600),
(np.random.normal(loc=0, scale=1, size=4444).astype("<f2"), 3000),
(np.random.normal(loc=0, scale=1, size=4444).astype("<f4"), 3000),
(np.random.normal(loc=0, scale=1, size=4444).astype("<f8"), 3000),
(np.random.choice([b"A", b"C", b"G", b"T"], size=5555, replace=True).astype("S"), 3000),
(
np.random.choice(["foo", "bar", "baz", "quux"], size=5555, replace=True).astype("<U"),
100,
3000,
),
(
np.random.choice([0, 1 / 3, 1 / 7, 1 / 9, np.nan], size=5555, replace=True).astype(
"<f8"
),
100,
3000,
),
(np.random.randint(0, 2, size=5555, dtype=bool), 100),
(np.arange(20000, dtype="<i4").reshape(2000, 10, order="C"), (100, 3)),
(np.arange(20000, dtype="<i4").reshape(200, 100, order="F"), (100, 30)),
(np.arange(20000, dtype="<i4").reshape(200, 10, 10, order="C"), (100, 3, 3)),
(np.arange(20000, dtype="<i4").reshape(20, 100, 10, order="F"), (10, 30, 3)),
(np.arange(20000, dtype="<i4").reshape(20, 10, 10, 10, order="C"), (10, 3, 3, 3)),
(np.arange(20000, dtype="<i4").reshape(20, 10, 10, 10, order="F"), (10, 3, 3, 3)),
(np.random.randint(0, 2, size=5555, dtype=bool), 3000),
(np.arange(20000, dtype="<i4").reshape(2000, 10, order="C"), (2000, 5)),
(np.arange(20000, dtype="<i4").reshape(200, 100, order="F"), (200, 50)),
(np.arange(20000, dtype="<i4").reshape(200, 10, 10, order="C"), (200, 10, 5)),
(np.arange(20000, dtype="<i4").reshape(20, 100, 10, order="F"), (20, 100, 5)),
(np.arange(20000, dtype="<i4").reshape(20, 10, 10, 10, order="C"), (20, 10, 10, 5)),
(np.arange(20000, dtype="<i4").reshape(20, 10, 10, 10, order="F"), (20, 10, 10, 5)),
]

compressors = [
Expand All @@ -2280,9 +2280,9 @@ def test_format_compatibility():
for j, compressor in enumerate(compressors):
path = "{}/{}".format(i, j)

if path not in fixture: # pragma: no cover
#if path not in fixture: # pragma: no cover
# store the data - should be one-time operation
fixture.array(path, data=arr, chunks=chunks, order=order, compressor=compressor)
# fixture.array(path, data=arr, chunks=chunks, order=order, compressor=compressor)

# setup array
z = fixture[path]
Expand Down Expand Up @@ -2457,13 +2457,12 @@ def test_normalize_store_arg(tmpdir):
assert isinstance(store, FSStore)


def test_meta_prefix_6853():
fixture = pathlib.Path(zarr.v2.__file__).resolve().parent.parent / "fixture"
def test_meta_prefix_6853(tmpdir):
fixture = pathlib.Path(str(tmpdir)) / "fixture"
meta = fixture / "meta"
if not meta.exists(): # pragma: no cover
s = DirectoryStore(str(meta), dimension_separator=".")
a = zarr.v2.open(store=s, mode="w", shape=(2, 2), dtype="<i8")
a[:] = [[1, 2], [3, 4]]
s = DirectoryStore(str(meta), dimension_separator=".")
a = zarr.v2.open(store=s, mode="w", shape=(2, 2), dtype="<i8")
a[:] = [[1, 2], [3, 4]]

fixtures = group(store=DirectoryStore(str(fixture)))
assert list(fixtures.arrays())
Expand Down