Skip to content

Clean up zarr.codecs namespace #2176

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
wants to merge 2 commits into from
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
29 changes: 21 additions & 8 deletions src/zarr/codecs/__init__.py
Original file line number Diff line number Diff line change
@@ -1,13 +1,14 @@
from __future__ import annotations

from zarr.codecs.blosc import BloscCname, BloscCodec, BloscShuffle
from zarr.codecs.bytes import BytesCodec, Endian
from zarr.codecs.crc32c_ import Crc32cCodec
from zarr.codecs.gzip import GzipCodec
from zarr.codecs.pipeline import BatchedCodecPipeline
from zarr.codecs.sharding import ShardingCodec, ShardingCodecIndexLocation
from zarr.codecs.transpose import TransposeCodec
from zarr.codecs.zstd import ZstdCodec
from zarr.codecs._blosc import BloscCname, BloscCodec, BloscShuffle
from zarr.codecs._bytes import BytesCodec, Endian
from zarr.codecs._crc32c_ import Crc32cCodec
from zarr.codecs._gzip import GzipCodec
from zarr.codecs._pipeline import BatchedCodecPipeline
from zarr.codecs._sharding import ShardingCodec, ShardingCodecIndexLocation
from zarr.codecs._transpose import TransposeCodec
from zarr.codecs._zstd import ZstdCodec
from zarr.registry import register_codec, register_pipeline

__all__ = [
"BatchedCodecPipeline",
Expand All @@ -23,3 +24,15 @@
"TransposeCodec",
"ZstdCodec",
]


register_codec("blosc", BloscCodec)
register_codec("bytes", BytesCodec)
# compatibility with earlier versions of ZEP1
register_codec("endian", BytesCodec)
register_codec("crc32c", Crc32cCodec)
register_codec("gzip", GzipCodec)
register_pipeline(BatchedCodecPipeline)
register_codec("sharding_indexed", ShardingCodec)
register_codec("transpose", TransposeCodec)
register_codec("zstd", ZstdCodec)
4 changes: 0 additions & 4 deletions src/zarr/codecs/blosc.py → src/zarr/codecs/_blosc.py
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,6 @@
from zarr.abc.codec import BytesBytesCodec
from zarr.core.buffer.cpu import as_numpy_array_wrapper
from zarr.core.common import JSON, parse_enum, parse_named_configuration, to_thread
from zarr.registry import register_codec

if TYPE_CHECKING:
from typing_extensions import Self
Expand Down Expand Up @@ -189,6 +188,3 @@ async def _encode_single(

def compute_encoded_size(self, _input_byte_length: int, _chunk_spec: ArraySpec) -> int:
raise NotImplementedError


register_codec("blosc", BloscCodec)
7 changes: 0 additions & 7 deletions src/zarr/codecs/bytes.py → src/zarr/codecs/_bytes.py
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,6 @@
from zarr.abc.codec import ArrayBytesCodec
from zarr.core.buffer import Buffer, NDArrayLike, NDBuffer
from zarr.core.common import JSON, parse_enum, parse_named_configuration
from zarr.registry import register_codec

if TYPE_CHECKING:
from typing_extensions import Self
Expand Down Expand Up @@ -119,9 +118,3 @@ async def _encode_single(

def compute_encoded_size(self, input_byte_length: int, _chunk_spec: ArraySpec) -> int:
return input_byte_length


register_codec("bytes", BytesCodec)

# compatibility with earlier versions of ZEP1
register_codec("endian", BytesCodec)
4 changes: 0 additions & 4 deletions src/zarr/codecs/crc32c_.py → src/zarr/codecs/_crc32c_.py
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,6 @@

from zarr.abc.codec import BytesBytesCodec
from zarr.core.common import JSON, parse_named_configuration
from zarr.registry import register_codec

if TYPE_CHECKING:
from typing_extensions import Self
Expand Down Expand Up @@ -61,6 +60,3 @@ async def _encode_single(

def compute_encoded_size(self, input_byte_length: int, _chunk_spec: ArraySpec) -> int:
return input_byte_length + 4


register_codec("crc32c", Crc32cCodec)
4 changes: 0 additions & 4 deletions src/zarr/codecs/gzip.py → src/zarr/codecs/_gzip.py
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,6 @@
from zarr.abc.codec import BytesBytesCodec
from zarr.core.buffer.cpu import as_numpy_array_wrapper
from zarr.core.common import JSON, parse_named_configuration, to_thread
from zarr.registry import register_codec

if TYPE_CHECKING:
from typing_extensions import Self
Expand Down Expand Up @@ -70,6 +69,3 @@ def compute_encoded_size(
_chunk_spec: ArraySpec,
) -> int:
raise NotImplementedError


register_codec("gzip", GzipCodec)
5 changes: 1 addition & 4 deletions src/zarr/codecs/pipeline.py → src/zarr/codecs/_pipeline.py
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@
from zarr.core.common import JSON, ChunkCoords, concurrent_map, parse_named_configuration
from zarr.core.config import config
from zarr.core.indexing import SelectorTuple, is_scalar, is_total_slice
from zarr.registry import get_codec_class, register_pipeline
from zarr.registry import get_codec_class

if TYPE_CHECKING:
import numpy as np
Expand Down Expand Up @@ -526,6 +526,3 @@ def codecs_from_list(
raise ValueError("Required ArrayBytesCodec was not found.")
else:
return array_array, array_bytes_maybe, bytes_bytes


register_pipeline(BatchedCodecPipeline)
9 changes: 3 additions & 6 deletions src/zarr/codecs/sharding.py → src/zarr/codecs/_sharding.py
Original file line number Diff line number Diff line change
Expand Up @@ -18,8 +18,8 @@
CodecPipeline,
)
from zarr.abc.store import ByteGetter, ByteSetter
from zarr.codecs.bytes import BytesCodec
from zarr.codecs.crc32c_ import Crc32cCodec
from zarr.codecs._bytes import BytesCodec
from zarr.codecs._crc32c_ import Crc32cCodec
from zarr.core.array_spec import ArraySpec
from zarr.core.buffer import (
Buffer,
Expand All @@ -45,7 +45,7 @@
morton_order_iter,
)
from zarr.core.metadata import parse_codecs
from zarr.registry import get_ndbuffer_class, get_pipeline_class, register_codec
from zarr.registry import get_ndbuffer_class, get_pipeline_class

if TYPE_CHECKING:
from collections.abc import Awaitable, Callable, Iterator
Expand Down Expand Up @@ -722,6 +722,3 @@ async def _load_full_shard_maybe(
def compute_encoded_size(self, input_byte_length: int, shard_spec: ArraySpec) -> int:
chunks_per_shard = self._get_chunks_per_shard(shard_spec)
return input_byte_length + self._shard_index_size(chunks_per_shard)


register_codec("sharding_indexed", ShardingCodec)
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,6 @@
from zarr.abc.codec import ArrayArrayCodec
from zarr.core.array_spec import ArraySpec
from zarr.core.common import JSON, ChunkCoordsLike, parse_named_configuration
from zarr.registry import register_codec

if TYPE_CHECKING:
from typing import Any
Expand Down Expand Up @@ -109,6 +108,3 @@ async def _encode_single(

def compute_encoded_size(self, input_byte_length: int, _chunk_spec: ArraySpec) -> int:
return input_byte_length


register_codec("transpose", TransposeCodec)
4 changes: 0 additions & 4 deletions src/zarr/codecs/zstd.py → src/zarr/codecs/_zstd.py
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,6 @@
from zarr.abc.codec import BytesBytesCodec
from zarr.core.buffer.cpu import as_numpy_array_wrapper
from zarr.core.common import JSON, parse_named_configuration, to_thread
from zarr.registry import register_codec

if TYPE_CHECKING:
from typing_extensions import Self
Expand Down Expand Up @@ -88,6 +87,3 @@ async def _encode_single(

def compute_encoded_size(self, _input_byte_length: int, _chunk_spec: ArraySpec) -> int:
raise NotImplementedError


register_codec("zstd", ZstdCodec)
Empty file removed src/zarr/codecs/registry.py
Empty file.
7 changes: 1 addition & 6 deletions tests/v3/test_buffer.py
Original file line number Diff line number Diff line change
Expand Up @@ -6,12 +6,7 @@
import pytest

from zarr import AsyncArray
from zarr.codecs.blosc import BloscCodec
from zarr.codecs.bytes import BytesCodec
from zarr.codecs.crc32c_ import Crc32cCodec
from zarr.codecs.gzip import GzipCodec
from zarr.codecs.transpose import TransposeCodec
from zarr.codecs.zstd import ZstdCodec
from zarr.codecs import BloscCodec, BytesCodec, Crc32cCodec, GzipCodec, TransposeCodec, ZstdCodec
from zarr.core.buffer import ArrayLike, BufferPrototype, NDArrayLike, cpu, gpu
from zarr.store.common import StorePath
from zarr.store.memory import MemoryStore
Expand Down
2 changes: 1 addition & 1 deletion tests/v3/test_metadata/test_v3.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@
import re
from typing import TYPE_CHECKING, Literal

from zarr.codecs.bytes import BytesCodec
from zarr.codecs import BytesCodec
from zarr.core.buffer import default_buffer_prototype
from zarr.core.chunk_key_encodings import DefaultChunkKeyEncoding, V2ChunkKeyEncoding

Expand Down
Loading