Merge branch 'v3' into fix-hypothesis-complex-warning

* v3:
  fix: opening a group with unspecified format finds either v2 or v3 (zarr-developers#2183)
  test: check that store, array, and group classes are serializable  (zarr-developers#2006)
  feature(store): V3 ZipStore (zarr-developers#2078)
  More typing fixes for tests (zarr-developers#2173)
  refactor: split metadata into v2 and v3 modules (zarr-developers#2163)
  Accept dictionaries for `store` argument (zarr-developers#2164)
  Simplify mypy config for tests (zarr-developers#2156)
  Fixed path segment duplication in open_array (zarr-developers#2167)
  Fixed test warnings (zarr-developers#2168)
  chore: update pre-commit hooks (zarr-developers#2165)
  Ensure that store_dict used for empty dicts (zarr-developers#2162)
  Bump pypa/gh-action-pypi-publish from 1.10.0 to 1.10.1 in the actions group (zarr-developers#2160)

Author: dcherian

.github/workflows/releases.yml

@@ -55,7 +55,7 @@ jobs:
         with:
           name: releases
           path: dist
-      - uses: pypa/gh-action-pypi-publish@v1.10.0
+      - uses: pypa/gh-action-pypi-publish@v1.10.1
         with:
           user: __token__
           password: ${{ secrets.pypi_password }}

.pre-commit-config.yaml

@@ -7,7 +7,7 @@ default_language_version:
   python: python3
 repos:
   - repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: v0.6.3
+    rev: v0.6.4
     hooks:
     - id: ruff
       args: ["--fix", "--show-fixes"]
@@ -25,7 +25,7 @@ repos:
     rev: v1.11.2
     hooks:
       - id: mypy
-        files: src|tests/v3/test_(api|array|buffer).py
+        files: src|tests
         additional_dependencies:
           # Package dependencies
           - asciitree
@@ -41,7 +41,7 @@ repos:
           # Zarr v2
           - types-redis
   - repo: https://github.com/scientific-python/cookie
-    rev: 2024.04.23
+    rev: 2024.08.19
     hooks:
       - id: sp-repo-review
   - repo: https://github.com/pre-commit/pygrep-hooks

pyproject.toml

@@ -181,7 +181,6 @@ serve = "sphinx-autobuild docs docs/_build --host 0.0.0.0"
 
 [tool.ruff]
 line-length = 100
-src = ["src"]
 force-exclude = true
 extend-exclude = [
     ".bzr",
@@ -226,6 +225,7 @@ python_version = "3.10"
 ignore_missing_imports = true
 namespace_packages = false
 
+
 strict = true
 warn_unreachable = true
 
@@ -237,6 +237,23 @@ module = [
 ]
 ignore_errors = true
 
+[[tool.mypy.overrides]]
+module = [
+    "tests.v2.*",
+    "tests.v3.package_with_entrypoint.*",
+    "tests.v3.test_codecs.test_codecs",
+    "tests.v3.test_codecs.test_transpose",
+    "tests.v3.test_metadata.*",
+    "tests.v3.test_store.*",
+    "tests.v3.test_config",
+    "tests.v3.test_group",
+    "tests.v3.test_indexing",
+    "tests.v3.test_properties",
+    "tests.v3.test_sync",
+    "tests.v3.test_v2",
+]
+ignore_errors = true
+
 [tool.pytest.ini_options]
 minversion = "7"
 testpaths = ["tests"]
@@ -263,6 +280,6 @@ markers = [
 
 [tool.repo-review]
 ignore = [
-	"PC111",  # fix Python code in documentation - enable later 
+	"PC111",  # fix Python code in documentation - enable later
 	"PC180",  # for JavaScript - not interested
 ]

src/zarr/abc/store.py

@@ -11,6 +11,7 @@
 
 
 class AccessMode(NamedTuple):
+    str: AccessModeLiteral
     readonly: bool
     overwrite: bool
     create: bool
@@ -20,6 +21,7 @@ class AccessMode(NamedTuple):
     def from_literal(cls, mode: AccessModeLiteral) -> Self:
         if mode in ("r", "r+", "a", "w", "w-"):
             return cls(
+                str=mode,
                 readonly=mode == "r",
                 overwrite=mode == "w",
                 create=mode in ("a", "w", "w-"),
@@ -42,6 +44,14 @@ async def open(cls, *args: Any, **kwargs: Any) -> Self:
         await store._open()
         return store
 
+    def __enter__(self) -> Self:
+        """Enter a context manager that will close the store upon exiting."""
+        return self
+
+    def __exit__(self, *args: Any) -> None:
+        """Close the store."""
+        self.close()
+
     async def _open(self) -> None:
         if self._is_open:
             raise ValueError("store is already open")
@@ -73,6 +83,11 @@ def _check_writable(self) -> None:
         if self.mode.readonly:
             raise ValueError("store mode does not support writing")
 
+    @abstractmethod
+    def __eq__(self, value: object) -> bool:
+        """Equality comparison."""
+        ...
+
     @abstractmethod
     async def get(
         self,
@@ -143,6 +158,12 @@ async def set(self, key: str, value: Buffer) -> None:
         """
         ...
 
+    @property
+    @abstractmethod
+    def supports_deletes(self) -> bool:
+        """Does the store support deletes?"""
+        ...
+
     @abstractmethod
     async def delete(self, key: str) -> None:
         """Remove a key from the store
@@ -221,7 +242,6 @@ def list_dir(self, prefix: str) -> AsyncGenerator[str, None]:
     def close(self) -> None:
         """Close the store."""
         self._is_open = False
-        pass
 
 
 @runtime_checkable

src/zarr/api/asynchronous.py

@@ -9,8 +9,10 @@
 
 from zarr.core.array import Array, AsyncArray
 from zarr.core.common import JSON, AccessModeLiteral, ChunkCoords, MemoryOrder, ZarrFormat
+from zarr.core.config import config
 from zarr.core.group import AsyncGroup
-from zarr.core.metadata import ArrayV2Metadata, ArrayV3Metadata
+from zarr.core.metadata.v2 import ArrayV2Metadata
+from zarr.core.metadata.v3 import ArrayV3Metadata
 from zarr.store import (
     StoreLike,
     make_store_path,
@@ -125,8 +127,7 @@ def _handle_zarr_version_or_format(
 
 def _default_zarr_version() -> ZarrFormat:
     """return the default zarr_version"""
-    # TODO: set default value from config
-    return 3
+    return cast(ZarrFormat, int(config.get("default_zarr_version", 3)))
 
 
 async def consolidate_metadata(*args: Any, **kwargs: Any) -> AsyncGroup:
@@ -336,7 +337,10 @@ async def save_group(
     kwargs
         NumPy arrays with data to save.
     """
-    zarr_format = _handle_zarr_version_or_format(zarr_version=zarr_version, zarr_format=zarr_format)
+    zarr_format = (
+        _handle_zarr_version_or_format(zarr_version=zarr_version, zarr_format=zarr_format)
+        or _default_zarr_version()
+    )
 
     if len(args) == 0 and len(kwargs) == 0:
         raise ValueError("at least one array must be provided")
@@ -447,10 +451,7 @@ async def group(
         The new group.
     """
 
-    zarr_format = (
-        _handle_zarr_version_or_format(zarr_version=zarr_version, zarr_format=zarr_format)
-        or _default_zarr_version()
-    )
+    zarr_format = _handle_zarr_version_or_format(zarr_version=zarr_version, zarr_format=zarr_format)
 
     store_path = await make_store_path(store)
     if path is not None:
@@ -473,7 +474,7 @@ async def group(
     except (KeyError, FileNotFoundError):
         return await AsyncGroup.create(
             store=store_path,
-            zarr_format=zarr_format,
+            zarr_format=zarr_format or _default_zarr_version(),
             exists_ok=overwrite,
             attributes=attributes,
         )
@@ -482,7 +483,7 @@ async def group(
 async def open_group(
     *,  # Note: this is a change from v2
     store: StoreLike | None = None,
-    mode: AccessModeLiteral | None = None,  # not used
+    mode: AccessModeLiteral | None = None,
     cache_attrs: bool | None = None,  # not used, default changed
     synchronizer: Any = None,  # not used
     path: str | None = None,
@@ -497,8 +498,18 @@ async def open_group(
 
     Parameters
     ----------
-    store : Store or string, optional
+    store : Store, string, or mapping, optional
         Store or path to directory in file system or name of zip file.
+
+        Strings are interpreted as paths on the local file system
+        and used as the ``root`` argument to :class:`zarr.store.LocalStore`.
+
+        Dictionaries are used as the ``store_dict`` argument in
+        :class:`zarr.store.MemoryStore``.
+
+        By default (``store=None``) a new :class:`zarr.store.MemoryStore`
+        is created.
+
     mode : {'r', 'r+', 'a', 'w', 'w-'}, optional
         Persistence mode: 'r' means read only (must exist); 'r+' means
         read/write (must exist); 'a' means read/write (create if doesn't
@@ -527,10 +538,7 @@ async def open_group(
         The new group.
     """
 
-    zarr_format = (
-        _handle_zarr_version_or_format(zarr_version=zarr_version, zarr_format=zarr_format)
-        or _default_zarr_version()
-    )
+    zarr_format = _handle_zarr_version_or_format(zarr_version=zarr_version, zarr_format=zarr_format)
 
     if cache_attrs is not None:
         warnings.warn("cache_attrs is not yet implemented", RuntimeWarning, stacklevel=2)
@@ -554,7 +562,10 @@ async def open_group(
         return await AsyncGroup.open(store_path, zarr_format=zarr_format)
     except (KeyError, FileNotFoundError):
         return await AsyncGroup.create(
-            store_path, zarr_format=zarr_format, exists_ok=True, attributes=attributes
+            store_path,
+            zarr_format=zarr_format or _default_zarr_version(),
+            exists_ok=True,
+            attributes=attributes,
         )
 
 
@@ -676,7 +687,7 @@ async def create(
 
     if zarr_format == 2 and chunks is None:
         chunks = shape
-    if zarr_format == 3 and chunk_shape is None:
+    elif zarr_format == 3 and chunk_shape is None:
         if chunks is not None:
             chunk_shape = chunks
             chunks = None
@@ -897,8 +908,7 @@ async def open_array(
         if store_path.store.mode.create:
             return await create(
                 store=store_path,
-                path=path,
-                zarr_format=zarr_format,
+                zarr_format=zarr_format or _default_zarr_version(),
                 overwrite=store_path.store.mode.overwrite,
                 **kwargs,
             )

src/zarr/codecs/sharding.py

@@ -44,7 +44,7 @@
     get_indexer,
     morton_order_iter,
 )
-from zarr.core.metadata import parse_codecs
+from zarr.core.metadata.v3 import parse_codecs
 from zarr.registry import get_ndbuffer_class, get_pipeline_class, register_codec
 
 if TYPE_CHECKING:

src/zarr/core/array.py

@@ -55,7 +55,8 @@
     is_scalar,
     pop_fields,
 )
-from zarr.core.metadata import ArrayMetadata, ArrayV2Metadata, ArrayV3Metadata
+from zarr.core.metadata.v2 import ArrayV2Metadata
+from zarr.core.metadata.v3 import ArrayV3Metadata
 from zarr.core.sync import sync
 from zarr.registry import get_pipeline_class
 from zarr.store import StoreLike, StorePath, make_store_path
@@ -67,6 +68,7 @@
     from collections.abc import Iterable
 
     from zarr.abc.codec import Codec, CodecPipeline
+    from zarr.core.metadata.common import ArrayMetadata
 
 # Array and AsyncArray are defined in the base ``zarr`` namespace
 __all__ = ["parse_array_metadata", "create_codec_pipeline"]

src/zarr/core/buffer/core.py

@@ -195,8 +195,8 @@ def from_buffer(cls, buffer: Buffer) -> Self:
         -------
             A new buffer representing the content of the input buffer
 
-        Note
-        ----
+        Notes
+        -----
         Subclasses of `Buffer` must override this method to implement
         more optimal conversions that avoid copies where possible
         """
@@ -281,6 +281,12 @@ def __add__(self, other: Buffer) -> Self:
         """Concatenate two buffers"""
         ...
 
+    def __eq__(self, other: object) -> bool:
+        # Another Buffer class can override this to choose a more efficient path
+        return isinstance(other, Buffer) and np.array_equal(
+            self.as_numpy_array(), other.as_numpy_array()
+        )
+
 
 class NDBuffer:
     """An n-dimensional memory block

src/zarr/core/buffer/cpu.py

@@ -73,8 +73,8 @@ def from_buffer(cls, buffer: core.Buffer) -> Self:
         -------
             A new buffer representing the content of the input buffer
 
-        Note
-        ----
+        Notes
+        -----
         Subclasses of `Buffer` must override this method to implement
         more optimal conversions that avoid copies where possible
         """

src/zarr/core/buffer/gpu.py

@@ -38,8 +38,8 @@ class Buffer(core.Buffer):
     array-like instance can be copied/converted to a regular Numpy array
     (host memory).
 
-    Note
-    ----
+    Notes
+    -----
     This buffer is untyped, so all indexing and sizes are in bytes.
 
     Parameters
@@ -123,8 +123,8 @@ class NDBuffer(core.NDBuffer):
     ndarray-like instance can be copied/converted to a regular Numpy array
     (host memory).
 
-    Note
-    ----
+    Notes
+    -----
     The two buffer classes Buffer and NDBuffer are very similar. In fact, Buffer
     is a special case of NDBuffer where dim=1, stride=1, and dtype="b". However,
     in order to use Python's type system to differentiate between the contiguous
@@ -193,8 +193,8 @@ def from_numpy_array(cls, array_like: npt.ArrayLike) -> Self:
     def as_numpy_array(self) -> npt.NDArray[Any]:
         """Returns the buffer as a NumPy array (host memory).
 
-        Warning
-        -------
+        Warnings
+        --------
         Might have to copy data, consider using `.as_ndarray_like()` instead.
 
         Returns