Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Use full config dict for default codecs #2584

Closed
wants to merge 42 commits into from
Closed
Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
42 commits
Select commit Hold shift + click to select a range
fd6ecd1
add functions for easy read-only data access
d-v-b Nov 4, 2024
fa343f5
sync funcs
d-v-b Nov 4, 2024
d95eba8
make read-only funcs top-level exports
d-v-b Nov 4, 2024
f6765bc
Merge branch 'main' into feat/read-funcs
d-v-b Nov 5, 2024
5d8445b
add create_array, create_group, and tests
d-v-b Nov 5, 2024
9526571
add top-level imports
d-v-b Nov 5, 2024
90bf421
Merge branch 'feat/read-funcs' of github.com:d-v-b/zarr-python into f…
d-v-b Nov 5, 2024
de280a7
add test for top-level exports
d-v-b Nov 5, 2024
d9878cf
add test for read
d-v-b Nov 5, 2024
e5217ce
add asserts
d-v-b Nov 5, 2024
40cc7af
Apply suggestions from code review
d-v-b Nov 5, 2024
d7ce58b
Merge branch 'main' into feat/read-funcs
d-v-b Nov 5, 2024
a0dfe18
Merge branch 'main' into feat/read-funcs
d-v-b Nov 5, 2024
98bc328
Merge branch 'main' into feat/read-funcs
d-v-b Nov 12, 2024
16f5cc2
Merge branch 'main' of github.com:zarr-developers/zarr-python into fe…
d-v-b Nov 29, 2024
4b45ebf
handle sharding in create_array
d-v-b Dec 10, 2024
750a439
Merge branch 'main' of github.com:zarr-developers/zarr-python into fe…
d-v-b Dec 10, 2024
7a5cbe7
tweak
d-v-b Dec 10, 2024
215ff96
Merge branch 'main' of github.com:zarr-developers/zarr-python into fe…
d-v-b Dec 18, 2024
489e2a2
make logic of _auto_partition better for shard shape
d-v-b Dec 18, 2024
05dd0d8
add dtype parsing, and tweak auto_partitioning func
d-v-b Dec 18, 2024
3fbfc21
sketch of docstring; remove auto chunks / shard shape
d-v-b Dec 19, 2024
b348737
Merge branch 'main' of github.com:zarr-developers/zarr-python into fe…
d-v-b Dec 19, 2024
5025ad6
tweak docstring
d-v-b Dec 19, 2024
e204a32
Merge branch 'main' of github.com:zarr-developers/zarr-python into fe…
d-v-b Dec 20, 2024
68465db
docstrings
d-v-b Dec 20, 2024
d7bb121
ensure tests pass
d-v-b Dec 20, 2024
99cc8f5
tuple -> list
d-v-b Dec 20, 2024
a39457f
allow data in create_array
d-v-b Dec 20, 2024
3f0a3e0
docstring
d-v-b Dec 20, 2024
26ced00
remove auto_partition
d-v-b Dec 20, 2024
af55ac4
make shape shapelike
d-v-b Dec 20, 2024
07f07ea
use create_array everywhere in group class
d-v-b Dec 20, 2024
bc552ce
remove readers
d-v-b Dec 20, 2024
74f731a
fix dodgy imports
d-v-b Dec 20, 2024
43877c0
compressors -> compression, auto chunking, auto sharding, auto compre…
d-v-b Dec 21, 2024
c693fb4
use sane shard shape when there are too few chunks
d-v-b Dec 21, 2024
4c18aaa
Merge branch 'main' into feat/read-funcs
jhamman Dec 22, 2024
dba2594
fix: allow user-specified filters and compression
d-v-b Dec 22, 2024
669ad72
np.dtype[np.generic] -> np.dtype[Any]
d-v-b Dec 22, 2024
ae1832d
handle singleton compressor / filters input
d-v-b Dec 22, 2024
5cb6dd8
default codec config now uses the full config dict
normanrz Dec 22, 2024
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
remove readers
  • Loading branch information
d-v-b committed Dec 20, 2024
commit bc552ce0305a7837a267f4d98cf889175e1f453c
6 changes: 0 additions & 6 deletions src/zarr/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -21,9 +21,6 @@
open_consolidated,
open_group,
open_like,
read,
read_array,
read_group,
save,
save_array,
save_group,
Expand Down Expand Up @@ -66,9 +63,6 @@
"open_consolidated",
"open_group",
"open_like",
"read",
"read_array",
"read_group",
"save",
"save_array",
"save_group",
Expand Down
102 changes: 0 additions & 102 deletions src/zarr/api/asynchronous.py
Original file line number Diff line number Diff line change
Expand Up @@ -339,47 +339,6 @@ async def open(
return await open_group(store=store_path, zarr_format=zarr_format, mode=mode, **kwargs)


async def read(
*,
store: StoreLike | None = None,
zarr_format: ZarrFormat | None = None,
path: str | None = None,
storage_options: dict[str, Any] | None = None,
**kwargs: Any,
) -> AsyncArray[ArrayV2Metadata] | AsyncArray[ArrayV3Metadata] | AsyncGroup:
"""Convenience function to open a group or array for reading. This function
wraps :func:`zarr.api.asynchronous.open` See the documentation of that function for details.

Parameters
----------
store : Store or str, optional
Store or path to directory in file system or name of zip file.
zarr_format : {2, 3, None}, optional
The zarr format to require. The default value of None will first look for Zarr v3 data,
then Zarr v2 data, then fail if neither format is found.
path : str or None, optional
The path within the store to open.
storage_options : dict, optional
If using an fsspec URL to create the store, this will be passed to
the backend implementation. Ignored otherwise.
**kwargs
Additional parameters are passed through to :func:`zarr.creation.open`.

Returns
-------
z : array or group
Return type depends on what exists in the given store.
"""
return await open(
store=store,
mode="r",
zarr_format=zarr_format,
path=path,
storage_options=storage_options,
**kwargs,
)


async def open_consolidated(
*args: Any, use_consolidated: Literal[True] = True, **kwargs: Any
) -> AsyncGroup:
Expand Down Expand Up @@ -867,67 +826,6 @@ async def open_group(
raise FileNotFoundError(f"Unable to find group: {store_path}")


async def read_group(
store: StoreLike,
*,
path: str | None = None,
zarr_format: ZarrFormat | None = None,
storage_options: dict[str, Any] | None = None,
use_consolidated: bool | str | None = None,
) -> AsyncGroup:
"""Open a group for reading. This function wraps :func:`zarr.api.asynchronous.open_group` See
the documentation of that function for details.

Parameters
----------
store : Store, str, or mapping, optional
Store or path to directory in file system or name of zip file.

Strings are interpreted as paths on the local file system
and used as the ``root`` argument to :class:`zarr.store.LocalStore`.

Dictionaries are used as the ``store_dict`` argument in
:class:`zarr.store.MemoryStore``.
path : str, optional
Group path within store.
zarr_format : {2, 3, None}, optional
The zarr format to require. The default value of None will first look for Zarr v3 data,
then Zarr v2 data, then fail if neither format is found.
storage_options : dict
If the store is backed by an fsspec-based implementation, then this dict will be passed to
the Store constructor for that implementation. Ignored otherwise.
use_consolidated : bool or str, default None
Whether to use consolidated metadata.

By default, consolidated metadata is used if it's present in the
store (in the ``zarr.json`` for Zarr v3 and in the ``.zmetadata`` file
for Zarr v2).

To explicitly require consolidated metadata, set ``use_consolidated=True``,
which will raise an exception if consolidated metadata is not found.

To explicitly *not* use consolidated metadata, set ``use_consolidated=False``,
which will fall back to using the regular, non consolidated metadata.

Zarr v2 allowed configuring the key storing the consolidated metadata
(``.zmetadata`` by default). Specify the custom key as ``use_consolidated``
to load consolidated metadata from a non-default key.

Returns
-------
g : group
The new group.
"""
return await open_group(
store=store,
mode="r",
path=path,
storage_options=storage_options,
zarr_format=zarr_format,
use_consolidated=use_consolidated,
)


async def create(
shape: ChunkCoords | int,
*, # Note: this is a change from v2
Expand Down
32 changes: 0 additions & 32 deletions src/zarr/api/synchronous.py
Original file line number Diff line number Diff line change
Expand Up @@ -190,16 +190,6 @@ def open(
return Group(obj)


def read(
store: StoreLike,
*,
zarr_format: ZarrFormat | None = None,
path: str | None = None,
**kwargs: Any,
) -> Array | Group:
return open(store=store, mode="r", zarr_format=zarr_format, path=path, **kwargs)


def open_consolidated(*args: Any, use_consolidated: Literal[True] = True, **kwargs: Any) -> Group:
"""
Alias for :func:`open_group` with ``use_consolidated=True``.
Expand Down Expand Up @@ -557,24 +547,6 @@ def create_group(
)


def read_group(
store: StoreLike,
*,
path: str | None = None,
storage_options: dict[str, Any] | None = None,
zarr_format: ZarrFormat | None = None,
use_consolidated: bool | str | None = None,
) -> Group:
return open_group(
store=store,
path=path,
mode="r",
zarr_format=zarr_format,
use_consolidated=use_consolidated,
storage_options=storage_options,
)


# TODO: add type annotations for kwargs
def create(
shape: ChunkCoords | int,
Expand Down Expand Up @@ -731,10 +703,6 @@ def create_array(*args: Any, **kwargs: Any) -> Array:
return Array(sync(zarr.core.array.create_array(*args, **kwargs)))


def read_array(*args: Any, **kwargs: Any) -> Array:
return Array(sync(async_api.read_array(*args, **kwargs)))


# TODO: add type annotations for kwargs
def empty(shape: ChunkCoords, **kwargs: Any) -> Array:
"""Create an empty array.
Expand Down
86 changes: 0 additions & 86 deletions tests/test_api.py
Original file line number Diff line number Diff line change
Expand Up @@ -19,9 +19,6 @@
load,
open,
open_group,
read,
read_array,
read_group,
save,
save_array,
save_group,
Expand Down Expand Up @@ -63,32 +60,6 @@ def test_create(memory_store: Store) -> None:
z = create(shape=(400, 100), chunks=(16, 16.5), store=store, overwrite=True) # type: ignore [arg-type]


@pytest.mark.parametrize("store", ["memory"], indirect=True)
def test_read(store: Store) -> None:
"""
Test that the polymorphic read function can return an Array or a Group, depending on the path argument.
"""
# create an array and a group
_ = create_group(store=store, path="group", attributes={"node_type": "group"})
_ = create_array(
store=store,
path="array",
shape=(10, 10),
chunk_shape=(1, 1),
dtype="uint8",
attributes={"node_type": "array"},
)

group_r = read(store, path="group")
assert isinstance(group_r, Group)
assert group_r.attrs == {"node_type": "group"}

array_r = read(store, path="array")
assert isinstance(array_r, Array)
assert array_r.attrs == {"node_type": "array"}
assert array_r.shape == (10, 10)


# TODO: parametrize over everything this function takes
@pytest.mark.parametrize("store", ["memory"], indirect=True)
def test_create_array(store: Store) -> None:
Expand All @@ -105,42 +76,6 @@ def test_create_array(store: Store) -> None:
assert np.array_equal(array_w[:], np.zeros(shape, dtype=array_w.dtype) + data_val)


@pytest.mark.parametrize("store", ["memory"], indirect=True)
def test_read_array(store: Store) -> None:
shape = (10, 10)
data_val = 1
path = "foo"

zarr_format: ZarrFormat

for zarr_format in (2, 3):
attrs = {"zarr_format": zarr_format}
node_w = create_array(
store,
path=path,
shape=shape,
attributes=attrs,
zarr_format=zarr_format,
chunk_shape=shape,
dtype="uint8",
)
node_w[:] = data_val

# check that the correct array can be read when both v2 and v3 arrays are present
for zarr_format in (2, 3):
node_r = read_array(store, path=path, zarr_format=zarr_format)

assert node_r.shape == shape
assert node_r.attrs == {"zarr_format": zarr_format}
assert np.array_equal(node_r[:], np.zeros(shape, dtype=node_r.dtype) + data_val)

# check that reading without specifying the zarr_format returns the v3 node
with pytest.warns(UserWarning):
node_r = read_array(store, path=path)

assert node_r.metadata.zarr_format == 3


@pytest.mark.parametrize("write_empty_chunks", [True, False])
def test_write_empty_chunks_warns(write_empty_chunks: bool) -> None:
"""
Expand Down Expand Up @@ -230,27 +165,6 @@ async def test_open_group(memory_store: MemoryStore) -> None:
assert g.read_only


@pytest.mark.parametrize("store", ["memory"], indirect=True)
def test_read_group(store: Store) -> None:
path = "foo"

zarr_format: ZarrFormat
for zarr_format in (2, 3):
attrs = {"zarr_format": zarr_format}
_ = create_group(store, path=path, attributes=attrs, zarr_format=zarr_format)

# check that the correct array can be read when both v2 and v3 arrays are present
for zarr_format in (2, 3):
node_r = read_group(store, path=path, zarr_format=zarr_format)
assert node_r.attrs == {"zarr_format": zarr_format}

# check that reading without specifying the zarr_format returns the v3 node
with pytest.warns(UserWarning):
node_r = read_group(store, path=path)

assert node_r.metadata.zarr_format == 3


@pytest.mark.parametrize("zarr_format", [None, 2, 3])
async def test_open_group_unspecified_version(
tmpdir: pathlib.Path, zarr_format: ZarrFormat
Expand Down
Loading