From 124640aef093ffdd6bf8d23e15d7559620d506df Mon Sep 17 00:00:00 2001 From: Joe Hamman Date: Fri, 11 Oct 2024 15:13:40 -0700 Subject: [PATCH] Revert "V3 main sync w/ merge (#2335)" (#2339) This reverts commit ce5f53e3482727b7133def725af24576057aa164. --- .github/ISSUE_TEMPLATE/bug_report.yml | 2 +- .github/workflows/releases.yml | 2 +- LICENSE.txt | 2 +- README.md | 6 +- TEAM.md | 29 -- docs/conf.py | 16 +- docs/index.rst | 4 +- docs/installation.rst | 5 - docs/release.rst | 33 -- docs/spec/v1.rst | 267 +++++++++++- docs/spec/v2.rst | 562 +++++++++++++++++++++++++- docs/spec/v3.rst | 2 +- docs/tutorial.rst | 40 +- pyproject.toml | 1 - v3-roadmap-and-design.md | 429 -------------------- 15 files changed, 851 insertions(+), 549 deletions(-) delete mode 100644 TEAM.md delete mode 100644 v3-roadmap-and-design.md diff --git a/.github/ISSUE_TEMPLATE/bug_report.yml b/.github/ISSUE_TEMPLATE/bug_report.yml index 705cd31cb5..2e11f14613 100644 --- a/.github/ISSUE_TEMPLATE/bug_report.yml +++ b/.github/ISSUE_TEMPLATE/bug_report.yml @@ -27,7 +27,7 @@ body: attributes: label: Python Version description: Version of Python interpreter - placeholder: 3.10, 3.11, 3.12 etc. + placeholder: 3.8.5, 3.9, 3.10, etc. validations: required: true - type: input diff --git a/.github/workflows/releases.yml b/.github/workflows/releases.yml index bab53958dc..f694039ce7 100644 --- a/.github/workflows/releases.yml +++ b/.github/workflows/releases.yml @@ -16,7 +16,7 @@ jobs: submodules: true fetch-depth: 0 - - uses: actions/setup-python@v5.2.0 + - uses: actions/setup-python@v5 name: Install Python with: python-version: '3.11' diff --git a/LICENSE.txt b/LICENSE.txt index a4de1c39d3..850a0d8772 100644 --- a/LICENSE.txt +++ b/LICENSE.txt @@ -1,6 +1,6 @@ The MIT License (MIT) -Copyright (c) 2015-2024 Zarr Developers +Copyright (c) 2015-2023 Zarr Developers Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal diff --git a/README.md b/README.md index e379c9719f..b035ffa597 100644 --- a/README.md +++ b/README.md @@ -70,10 +70,10 @@ - Zulip + Gitter - - + + diff --git a/TEAM.md b/TEAM.md deleted file mode 100644 index e6975d7c04..0000000000 --- a/TEAM.md +++ /dev/null @@ -1,29 +0,0 @@ -## Active core-developers -- @joshmoore (Josh Moore) -- @jni (Juan Nunez-Iglesias) -- @rabernat (Ryan Abernathey) -- @jhamman (Joe Hamman) -- @d-v-b (Davis Bennett) -- @jakirkham (jakirkham) -- @martindurant (Martin Durant) -- @normanrz (Norman Rzepka) -- @dstansby (David Stansby) -- @dcherian (Deepak Cherian) -- @TomAugspurger (Tom Augspurger) - -## Emeritus core-developers -- @alimanfoo (Alistair Miles) -- @shoyer (Stephan Hoyer) -- @ryan-williams (Ryan Williams) -- @jrbourbeau (James Bourbeau) -- @mzjp2 (Zain Patel) -- @grlee77 (Gregory Lee) - -## Former core-developers -- @jeromekelleher (Jerome Kelleher) -- @tjcrone (Tim Crone) -- @funkey (Jan Funke) -- @shikharsg -- @Carreau (Matthias Bussonnier) -- @dazzag24 -- @WardF (Ward Fisher) diff --git a/docs/conf.py b/docs/conf.py index 0a328ac25f..ec85098372 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -19,8 +19,11 @@ import sphinx.application -from importlib.metadata import version as get_version - +# The version info for the project you're documenting, acts as replacement for +# |version| and |release|, also used in various other places throughout the +# built documents. +# +# The short X.Y version. import zarr import sphinx @@ -76,11 +79,12 @@ # General information about the project. project = "zarr" -copyright = "2024, Zarr Developers" +copyright = "2023, Zarr Developers" author = "Zarr Developers" -version = get_version("zarr") -release = get_version("zarr") +version = zarr.__version__ +# The full version, including alpha/beta/rc tags. +release = zarr.__version__ # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. @@ -201,7 +205,7 @@ def setup(app: sphinx.application.Sphinx) -> None: # html_use_smartypants = True # Custom sidebar templates, maps document names to template names. -html_sidebars = {"tutorial": []} +# html_sidebars = {} # Additional templates that should be rendered to pages, maps page names to # template names. diff --git a/docs/index.rst b/docs/index.rst index 6b90b5a773..8e1b0740cf 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -19,13 +19,13 @@ Zarr-Python **Version**: |version| -**Download documentation**: `PDF/Zipped HTML `_ +**Download documentation**: `PDF/Zipped HTML/EPUB `_ **Useful links**: `Installation `_ | `Source Repository `_ | `Issue Tracker `_ | -`Zulip Chat `_ +`Gitter `_ Zarr is a file storage format for chunked, compressed, N-dimensional arrays based on an open-source specification. diff --git a/docs/installation.rst b/docs/installation.rst index 86da6d1035..3d4ac41072 100644 --- a/docs/installation.rst +++ b/docs/installation.rst @@ -6,11 +6,6 @@ Zarr depends on NumPy. It is generally best to `install NumPy appropriate for your operating system and Python distribution. Other dependencies should be installed automatically if using one of the installation methods below. -Note: Zarr has endorsed `Scientific-Python SPEC 0 `_ and now follows the version support window as outlined below: - -- Python: 36 months after initial release -- Core package dependencies (e.g. NumPy): 24 months after initial release - Install Zarr from PyPI:: $ pip install zarr diff --git a/docs/release.rst b/docs/release.rst index 0b6775c4a6..5c3a43a148 100644 --- a/docs/release.rst +++ b/docs/release.rst @@ -303,39 +303,6 @@ Documentation * doc: update release notes for 3.0.0.alpha. By :user:`Joe Hamman ` :issue:`1959`. -.. _release_2.18.3: - -2.18.3 ------- - -Enhancements -~~~~~~~~~~~~ -* Added support for creating a copy of data when converting a `zarr.Array` - to a numpy array. - By :user:`David Stansby ` (:issue:`2106`) and - :user:`Joe Hamman ` (:issue:`2123`). - -Maintenance -~~~~~~~~~~~ -* Removed support for Python 3.9. - By :user:`David Stansby ` (:issue:`2074`). - -* Fix a regression when using orthogonal indexing with a scalar. - By :user:`Deepak Cherian ` :issue:`1931` - -* Added compatibility with NumPy 2.1. - By :user:`David Stansby ` - -* Bump minimum NumPy version to 1.24. - :user:`Joe Hamman ` (:issue:`2127`). - -Deprecations -~~~~~~~~~~~~ - -* Deprecate :class:`zarr.n5.N5Store` and :class:`zarr.n5.N5FSStore`. These - stores are slated to be removed in Zarr Python 3.0. - By :user:`Joe Hamman ` :issue:`2085`. - .. _release_2.18.2: 2.18.2 diff --git a/docs/spec/v1.rst b/docs/spec/v1.rst index 27a0490e0a..13f68ef36e 100644 --- a/docs/spec/v1.rst +++ b/docs/spec/v1.rst @@ -3,5 +3,268 @@ Zarr Storage Specification Version 1 ==================================== -The V1 Specification has been migrated to its website → -https://zarr-specs.readthedocs.io/. +This document provides a technical specification of the protocol and +format used for storing a Zarr array. The key words "MUST", "MUST +NOT", "REQUIRED", "SHALL", "SHALL NOT", "SHOULD", "SHOULD NOT", +"RECOMMENDED", "MAY", and "OPTIONAL" in this document are to be +interpreted as described in `RFC 2119 +`_. + +Status +------ + +This specification is deprecated. See :ref:`spec` for the latest version. + +Storage +------- + +A Zarr array can be stored in any storage system that provides a +key/value interface, where a key is an ASCII string and a value is an +arbitrary sequence of bytes, and the supported operations are read +(get the sequence of bytes associated with a given key), write (set +the sequence of bytes associated with a given key) and delete (remove +a key/value pair). + +For example, a directory in a file system can provide this interface, +where keys are file names, values are file contents, and files can be +read, written or deleted via the operating system. Equally, an S3 +bucket can provide this interface, where keys are resource names, +values are resource contents, and resources can be read, written or +deleted via HTTP. + +Below an "array store" refers to any system implementing this +interface. + +Metadata +-------- + +Each array requires essential configuration metadata to be stored, +enabling correct interpretation of the stored data. This metadata is +encoded using JSON and stored as the value of the 'meta' key within an +array store. + +The metadata resource is a JSON object. The following keys MUST be +present within the object: + +zarr_format + An integer defining the version of the storage specification to which the + array store adheres. +shape + A list of integers defining the length of each dimension of the array. +chunks + A list of integers defining the length of each dimension of a chunk of the + array. Note that all chunks within a Zarr array have the same shape. +dtype + A string or list defining a valid data type for the array. See also + the subsection below on data type encoding. +compression + A string identifying the primary compression library used to compress + each chunk of the array. +compression_opts + An integer, string or dictionary providing options to the primary + compression library. +fill_value + A scalar value providing the default value to use for uninitialized + portions of the array. +order + Either 'C' or 'F', defining the layout of bytes within each chunk of the + array. 'C' means row-major order, i.e., the last dimension varies fastest; + 'F' means column-major order, i.e., the first dimension varies fastest. + +Other keys MAY be present within the metadata object however they MUST +NOT alter the interpretation of the required fields defined above. + +For example, the JSON object below defines a 2-dimensional array of +64-bit little-endian floating point numbers with 10000 rows and 10000 +columns, divided into chunks of 1000 rows and 1000 columns (so there +will be 100 chunks in total arranged in a 10 by 10 grid). Within each +chunk the data are laid out in C contiguous order, and each chunk is +compressed using the Blosc compression library:: + + { + "chunks": [ + 1000, + 1000 + ], + "compression": "blosc", + "compression_opts": { + "clevel": 5, + "cname": "lz4", + "shuffle": 1 + }, + "dtype": "`_. The +format consists of 3 parts: a character describing the byteorder of +the data (``<``: little-endian, ``>``: big-endian, ``|``: +not-relevant), a character code giving the basic type of the array, +and an integer providing the number of bytes the type uses. The byte +order MUST be specified. E.g., ``"i4"``, ``"|b1"`` and +``"|S12"`` are valid data types. + +Structure data types (i.e., with multiple named fields) are encoded as +a list of two-element lists, following `NumPy array protocol type +descriptions (descr) +`_. +For example, the JSON list ``[["r", "|u1"], ["g", "|u1"], ["b", +"|u1"]]`` defines a data type composed of three single-byte unsigned +integers labelled 'r', 'g' and 'b'. + +Chunks +------ + +Each chunk of the array is compressed by passing the raw bytes for the +chunk through the primary compression library to obtain a new sequence +of bytes comprising the compressed chunk data. No header is added to +the compressed bytes or any other modification made. The internal +structure of the compressed bytes will depend on which primary +compressor was used. For example, the `Blosc compressor +`_ +produces a sequence of bytes that begins with a 16-byte header +followed by compressed data. + +The compressed sequence of bytes for each chunk is stored under a key +formed from the index of the chunk within the grid of chunks +representing the array. To form a string key for a chunk, the indices +are converted to strings and concatenated with the period character +('.') separating each index. For example, given an array with shape +(10000, 10000) and chunk shape (1000, 1000) there will be 100 chunks +laid out in a 10 by 10 grid. The chunk with indices (0, 0) provides +data for rows 0-999 and columns 0-999 and is stored under the key +'0.0'; the chunk with indices (2, 4) provides data for rows 2000-2999 +and columns 4000-4999 and is stored under the key '2.4'; etc. + +There is no need for all chunks to be present within an array +store. If a chunk is not present then it is considered to be in an +uninitialized state. An uninitialized chunk MUST be treated as if it +was uniformly filled with the value of the 'fill_value' field in the +array metadata. If the 'fill_value' field is ``null`` then the +contents of the chunk are undefined. + +Note that all chunks in an array have the same shape. If the length of +any array dimension is not exactly divisible by the length of the +corresponding chunk dimension then some chunks will overhang the edge +of the array. The contents of any chunk region falling outside the +array are undefined. + +Attributes +---------- + +Each array can also be associated with custom attributes, which are +simple key/value items with application-specific meaning. Custom +attributes are encoded as a JSON object and stored under the 'attrs' +key within an array store. Even if the attributes are empty, the +'attrs' key MUST be present within an array store. + +For example, the JSON object below encodes three attributes named +'foo', 'bar' and 'baz':: + + { + "foo": 42, + "bar": "apples", + "baz": [1, 2, 3, 4] + } + +Example +------- + +Below is an example of storing a Zarr array, using a directory on the +local file system as storage. + +Initialize the store:: + + >>> import zarr + >>> store = zarr.DirectoryStore('example.zarr') + >>> zarr.init_store(store, shape=(20, 20), chunks=(10, 10), + ... dtype='i4', fill_value=42, compression='zlib', + ... compression_opts=1, overwrite=True) + +No chunks are initialized yet, so only the 'meta' and 'attrs' keys +have been set:: + + >>> import os + >>> sorted(os.listdir('example.zarr')) + ['attrs', 'meta'] + +Inspect the array metadata:: + + >>> print(open('example.zarr/meta').read()) + { + "chunks": [ + 10, + 10 + ], + "compression": "zlib", + "compression_opts": 1, + "dtype": ">> print(open('example.zarr/attrs').read()) + {} + +Set some data:: + + >>> z = zarr.Array(store) + >>> z[0:10, 0:10] = 1 + >>> sorted(os.listdir('example.zarr')) + ['0.0', 'attrs', 'meta'] + +Set some more data:: + + >>> z[0:10, 10:20] = 2 + >>> z[10:20, :] = 3 + >>> sorted(os.listdir('example.zarr')) + ['0.0', '0.1', '1.0', '1.1', 'attrs', 'meta'] + +Manually decompress a single chunk for illustration:: + + >>> import zlib + >>> b = zlib.decompress(open('example.zarr/0.0', 'rb').read()) + >>> import numpy as np + >>> a = np.frombuffer(b, dtype='>> a + array([1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1], dtype=int32) + +Modify the array attributes:: + + >>> z.attrs['foo'] = 42 + >>> z.attrs['bar'] = 'apples' + >>> z.attrs['baz'] = [1, 2, 3, 4] + >>> print(open('example.zarr/attrs').read()) + { + "bar": "apples", + "baz": [ + 1, + 2, + 3, + 4 + ], + "foo": 42 + } diff --git a/docs/spec/v2.rst b/docs/spec/v2.rst index deb6d46ce6..c1e12e1218 100644 --- a/docs/spec/v2.rst +++ b/docs/spec/v2.rst @@ -3,5 +3,563 @@ Zarr Storage Specification Version 2 ==================================== -The V2 Specification has been migrated to its website → -https://zarr-specs.readthedocs.io/. +This document provides a technical specification of the protocol and format +used for storing Zarr arrays. The key words "MUST", "MUST NOT", "REQUIRED", +"SHALL", "SHALL NOT", "SHOULD", "SHOULD NOT", "RECOMMENDED", "MAY", and +"OPTIONAL" in this document are to be interpreted as described in `RFC 2119 +`_. + +Status +------ + +This specification is the latest version. See :ref:`spec` for previous +versions. + +.. _spec_v2_storage: + +Storage +------- + +A Zarr array can be stored in any storage system that provides a key/value +interface, where a key is an ASCII string and a value is an arbitrary sequence +of bytes, and the supported operations are read (get the sequence of bytes +associated with a given key), write (set the sequence of bytes associated with +a given key) and delete (remove a key/value pair). + +For example, a directory in a file system can provide this interface, where +keys are file names, values are file contents, and files can be read, written +or deleted via the operating system. Equally, an S3 bucket can provide this +interface, where keys are resource names, values are resource contents, and +resources can be read, written or deleted via HTTP. + +Below an "array store" refers to any system implementing this interface. + +.. _spec_v2_array: + +Arrays +------ + +.. _spec_v2_array_metadata: + +Metadata +~~~~~~~~ + +Each array requires essential configuration metadata to be stored, enabling +correct interpretation of the stored data. This metadata is encoded using JSON +and stored as the value of the ".zarray" key within an array store. + +The metadata resource is a JSON object. The following keys MUST be present +within the object: + +zarr_format + An integer defining the version of the storage specification to which the + array store adheres. +shape + A list of integers defining the length of each dimension of the array. +chunks + A list of integers defining the length of each dimension of a chunk of the + array. Note that all chunks within a Zarr array have the same shape. +dtype + A string or list defining a valid data type for the array. See also + the subsection below on data type encoding. +compressor + A JSON object identifying the primary compression codec and providing + configuration parameters, or ``null`` if no compressor is to be used. + The object MUST contain an ``"id"`` key identifying the codec to be used. +fill_value + A scalar value providing the default value to use for uninitialized + portions of the array, or ``null`` if no fill_value is to be used. +order + Either "C" or "F", defining the layout of bytes within each chunk of the + array. "C" means row-major order, i.e., the last dimension varies fastest; + "F" means column-major order, i.e., the first dimension varies fastest. +filters + A list of JSON objects providing codec configurations, or ``null`` if no + filters are to be applied. Each codec configuration object MUST contain a + ``"id"`` key identifying the codec to be used. + +The following keys MAY be present within the object: + +dimension_separator + If present, either the string ``"."`` or ``"/"`` defining the separator placed + between the dimensions of a chunk. If the value is not set, then the + default MUST be assumed to be ``"."``, leading to chunk keys of the form "0.0". + Arrays defined with ``"/"`` as the dimension separator can be considered to have + nested, or hierarchical, keys of the form "0/0" that SHOULD where possible + produce a directory-like structure. + +Other keys SHOULD NOT be present within the metadata object and SHOULD be +ignored by implementations. + +For example, the JSON object below defines a 2-dimensional array of 64-bit +little-endian floating point numbers with 10000 rows and 10000 columns, divided +into chunks of 1000 rows and 1000 columns (so there will be 100 chunks in total +arranged in a 10 by 10 grid). Within each chunk the data are laid out in C +contiguous order. Each chunk is encoded using a delta filter and compressed +using the Blosc compression library prior to storage:: + + { + "chunks": [ + 1000, + 1000 + ], + "compressor": { + "id": "blosc", + "cname": "lz4", + "clevel": 5, + "shuffle": 1 + }, + "dtype": "`. The format +consists of 3 parts: + +* One character describing the byteorder of the data (``"<"``: little-endian; + ``">"``: big-endian; ``"|"``: not-relevant) +* One character code giving the basic type of the array (``"b"``: Boolean (integer + type where all values are only True or False); ``"i"``: integer; ``"u"``: unsigned + integer; ``"f"``: floating point; ``"c"``: complex floating point; ``"m"``: timedelta; + ``"M"``: datetime; ``"S"``: string (fixed-length sequence of char); ``"U"``: unicode + (fixed-length sequence of Py_UNICODE); ``"V"``: other (void * – each item is a + fixed-size chunk of memory)) +* An integer specifying the number of bytes the type uses. + +The byte order MUST be specified. E.g., ``"i4"``, ``"|b1"`` and +``"|S12"`` are valid data type encodings. + +For datetime64 ("M") and timedelta64 ("m") data types, these MUST also include the +units within square brackets. A list of valid units and their definitions are given in +the :ref:`NumPy documentation on Datetimes and Timedeltas +`. +For example, ``"`. Each +sub-list has the form ``[fieldname, datatype, shape]`` where ``shape`` +is optional. ``fieldname`` is a string, ``datatype`` is a string +specifying a simple data type (see above), and ``shape`` is a list of +integers specifying subarray shape. For example, the JSON list below +defines a data type composed of three single-byte unsigned integer +fields named "r", "g" and "b":: + + [["r", "|u1"], ["g", "|u1"], ["b", "|u1"]] + +For example, the JSON list below defines a data type composed of three +fields named "x", "y" and "z", where "x" and "y" each contain 32-bit +floats, and each item in "z" is a 2 by 2 array of floats:: + + [["x", "`_ +produces a sequence of bytes that begins with a 16-byte header followed by +compressed data. + +The compressed sequence of bytes for each chunk is stored under a key formed +from the index of the chunk within the grid of chunks representing the array. +To form a string key for a chunk, the indices are converted to strings and +concatenated with the period character (".") separating each index. For +example, given an array with shape (10000, 10000) and chunk shape (1000, 1000) +there will be 100 chunks laid out in a 10 by 10 grid. The chunk with indices +(0, 0) provides data for rows 0-999 and columns 0-999 and is stored under the +key "0.0"; the chunk with indices (2, 4) provides data for rows 2000-2999 and +columns 4000-4999 and is stored under the key "2.4"; etc. + +There is no need for all chunks to be present within an array store. If a chunk +is not present then it is considered to be in an uninitialized state. An +uninitialized chunk MUST be treated as if it was uniformly filled with the value +of the "fill_value" field in the array metadata. If the "fill_value" field is +``null`` then the contents of the chunk are undefined. + +Note that all chunks in an array have the same shape. If the length of any +array dimension is not exactly divisible by the length of the corresponding +chunk dimension then some chunks will overhang the edge of the array. The +contents of any chunk region falling outside the array are undefined. + +.. _spec_v2_array_filters: + +Filters +~~~~~~~ + +Optionally a sequence of one or more filters can be used to transform chunk +data prior to compression. When storing data, filters are applied in the order +specified in array metadata to encode data, then the encoded data are passed to +the primary compressor. When retrieving data, stored chunk data are +decompressed by the primary compressor then decoded using filters in the +reverse order. + +.. _spec_v2_hierarchy: + +Hierarchies +----------- + +.. _spec_v2_hierarchy_paths: + +Logical storage paths +~~~~~~~~~~~~~~~~~~~~~ + +Multiple arrays can be stored in the same array store by associating each array +with a different logical path. A logical path is simply an ASCII string. The +logical path is used to form a prefix for keys used by the array. For example, +if an array is stored at logical path "foo/bar" then the array metadata will be +stored under the key "foo/bar/.zarray", the user-defined attributes will be +stored under the key "foo/bar/.zattrs", and the chunks will be stored under +keys like "foo/bar/0.0", "foo/bar/0.1", etc. + +To ensure consistent behaviour across different storage systems, logical paths +MUST be normalized as follows: + +* Replace all backward slash characters ("\\\\") with forward slash characters + ("/") +* Strip any leading "/" characters +* Strip any trailing "/" characters +* Collapse any sequence of more than one "/" character into a single "/" + character + +The key prefix is then obtained by appending a single "/" character to the +normalized logical path. + +After normalization, if splitting a logical path by the "/" character results +in any path segment equal to the string "." or the string ".." then an error +MUST be raised. + +N.B., how the underlying array store processes requests to store values under +keys containing the "/" character is entirely up to the store implementation +and is not constrained by this specification. E.g., an array store could simply +treat all keys as opaque ASCII strings; equally, an array store could map +logical paths onto some kind of hierarchical storage (e.g., directories on a +file system). + +.. _spec_v2_hierarchy_groups: + +Groups +~~~~~~ + +Arrays can be organized into groups which can also contain other groups. A +group is created by storing group metadata under the ".zgroup" key under some +logical path. E.g., a group exists at the root of an array store if the +".zgroup" key exists in the store, and a group exists at logical path "foo/bar" +if the "foo/bar/.zgroup" key exists in the store. + +If the user requests a group to be created under some logical path, then groups +MUST also be created at all ancestor paths. E.g., if the user requests group +creation at path "foo/bar" then groups MUST be created at path "foo" and the +root of the store, if they don't already exist. + +If the user requests an array to be created under some logical path, then +groups MUST also be created at all ancestor paths. E.g., if the user requests +array creation at path "foo/bar/baz" then groups must be created at path +"foo/bar", path "foo", and the root of the store, if they don't already exist. + +The group metadata resource is a JSON object. The following keys MUST be present +within the object: + +zarr_format + An integer defining the version of the storage specification to which the + array store adheres. + +Other keys MUST NOT be present within the metadata object. + +The members of a group are arrays and groups stored under logical paths that +are direct children of the parent group's logical path. E.g., if groups exist +under the logical paths "foo" and "foo/bar" and an array exists at logical path +"foo/baz" then the members of the group at path "foo" are the group at path +"foo/bar" and the array at path "foo/baz". + +.. _spec_v2_attrs: + +Attributes +---------- + +An array or group can be associated with custom attributes, which are arbitrary +key/value pairs with application-specific meaning. Custom attributes are encoded +as a JSON object and stored under the ".zattrs" key within an array store. The +".zattrs" key does not have to be present, and if it is absent the attributes +should be treated as empty. + +For example, the JSON object below encodes three attributes named +"foo", "bar" and "baz":: + + { + "foo": 42, + "bar": "apples", + "baz": [1, 2, 3, 4] + } + +.. _spec_v2_examples: + +Examples +-------- + +Storing a single array +~~~~~~~~~~~~~~~~~~~~~~ + +Below is an example of storing a Zarr array, using a directory on the +local file system as storage. + +Create an array:: + + >>> import zarr + >>> store = zarr.DirectoryStore('data/example.zarr') + >>> a = zarr.create(shape=(20, 20), chunks=(10, 10), dtype='i4', + ... fill_value=42, compressor=zarr.Zlib(level=1), + ... store=store, overwrite=True) + +No chunks are initialized yet, so only the ".zarray" and ".zattrs" keys +have been set in the store:: + + >>> import os + >>> sorted(os.listdir('data/example.zarr')) + ['.zarray'] + +Inspect the array metadata:: + + >>> print(open('data/example.zarr/.zarray').read()) + { + "chunks": [ + 10, + 10 + ], + "compressor": { + "id": "zlib", + "level": 1 + }, + "dtype": ">> a[0:10, 0:10] = 1 + >>> sorted(os.listdir('data/example.zarr')) + ['.zarray', '0.0'] + +Set some more data:: + + >>> a[0:10, 10:20] = 2 + >>> a[10:20, :] = 3 + >>> sorted(os.listdir('data/example.zarr')) + ['.zarray', '0.0', '0.1', '1.0', '1.1'] + +Manually decompress a single chunk for illustration:: + + >>> import zlib + >>> buf = zlib.decompress(open('data/example.zarr/0.0', 'rb').read()) + >>> import numpy as np + >>> chunk = np.frombuffer(buf, dtype='>> chunk + array([1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1], dtype=int32) + +Modify the array attributes:: + + >>> a.attrs['foo'] = 42 + >>> a.attrs['bar'] = 'apples' + >>> a.attrs['baz'] = [1, 2, 3, 4] + >>> sorted(os.listdir('data/example.zarr')) + ['.zarray', '.zattrs', '0.0', '0.1', '1.0', '1.1'] + >>> print(open('data/example.zarr/.zattrs').read()) + { + "bar": "apples", + "baz": [ + 1, + 2, + 3, + 4 + ], + "foo": 42 + } + +Storing multiple arrays in a hierarchy +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Below is an example of storing multiple Zarr arrays organized into a group +hierarchy, using a directory on the local file system as storage. This storage +implementation maps logical paths onto directory paths on the file system, +however this is an implementation choice and is not required. + +Setup the store:: + + >>> import zarr + >>> store = zarr.DirectoryStore('data/group.zarr') + +Create the root group:: + + >>> root_grp = zarr.group(store, overwrite=True) + +The metadata resource for the root group has been created:: + + >>> import os + >>> sorted(os.listdir('data/group.zarr')) + ['.zgroup'] + +Inspect the group metadata:: + + >>> print(open('data/group.zarr/.zgroup').read()) + { + "zarr_format": 2 + } + +Create a sub-group:: + + >>> sub_grp = root_grp.create_group('foo') + +What has been stored:: + + >>> sorted(os.listdir('data/group.zarr')) + ['.zgroup', 'foo'] + >>> sorted(os.listdir('data/group.zarr/foo')) + ['.zgroup'] + +Create an array within the sub-group:: + + >>> a = sub_grp.create_dataset('bar', shape=(20, 20), chunks=(10, 10)) + >>> a[:] = 42 + +Set a custom attributes:: + + >>> a.attrs['comment'] = 'answer to life, the universe and everything' + +What has been stored:: + + >>> sorted(os.listdir('data/group.zarr')) + ['.zgroup', 'foo'] + >>> sorted(os.listdir('data/group.zarr/foo')) + ['.zgroup', 'bar'] + >>> sorted(os.listdir('data/group.zarr/foo/bar')) + ['.zarray', '.zattrs', '0.0', '0.1', '1.0', '1.1'] + +Here is the same example using a Zip file as storage:: + + >>> store = zarr.ZipStore('data/group.zip', mode='w') + >>> root_grp = zarr.group(store) + >>> sub_grp = root_grp.create_group('foo') + >>> a = sub_grp.create_dataset('bar', shape=(20, 20), chunks=(10, 10)) + >>> a[:] = 42 + >>> a.attrs['comment'] = 'answer to life, the universe and everything' + >>> store.close() + +What has been stored:: + + >>> import zipfile + >>> zf = zipfile.ZipFile('data/group.zip', mode='r') + >>> for name in sorted(zf.namelist()): + ... print(name) + .zgroup + foo/.zgroup + foo/bar/.zarray + foo/bar/.zattrs + foo/bar/0.0 + foo/bar/0.1 + foo/bar/1.0 + foo/bar/1.1 + +.. _spec_v2_changes: + +Changes +------- + +Version 2 clarifications +~~~~~~~~~~~~~~~~~~~~~~~~ + +The following changes have been made to the version 2 specification since it was +initially published to clarify ambiguities and add some missing information. + +* The specification now describes how bytes fill values should be encoded and + decoded for arrays with a fixed-length byte string data type (:issue:`165`, + :issue:`176`). + +* The specification now clarifies that units must be specified for datetime64 and + timedelta64 data types (:issue:`85`, :issue:`215`). + +* The specification now clarifies that the '.zattrs' key does not have to be present for + either arrays or groups, and if absent then custom attributes should be treated as + empty. + +* The specification now describes how structured datatypes with + subarray shapes and/or with nested structured data types are encoded + in array metadata (:issue:`111`, :issue:`296`). + +* Clarified the key/value pairs of custom attributes as "arbitrary" rather than + "simple". + +Changes from version 1 to version 2 +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +The following changes were made between version 1 and version 2 of this specification: + +* Added support for storing multiple arrays in the same store and organising + arrays into hierarchies using groups. +* Array metadata is now stored under the ".zarray" key instead of the "meta" + key. +* Custom attributes are now stored under the ".zattrs" key instead of the + "attrs" key. +* Added support for filters. +* Changed encoding of "fill_value" field within array metadata. +* Changed encoding of compressor information within array metadata to be + consistent with representation of filter information. diff --git a/docs/spec/v3.rst b/docs/spec/v3.rst index 3d39f35ba6..bd8852707b 100644 --- a/docs/spec/v3.rst +++ b/docs/spec/v3.rst @@ -1,7 +1,7 @@ .. _spec_v3: Zarr Storage Specification Version 3 -==================================== +======================================================= The V3 Specification has been migrated to its website → https://zarr-specs.readthedocs.io/. diff --git a/docs/tutorial.rst b/docs/tutorial.rst index 619392a175..a40422490b 100644 --- a/docs/tutorial.rst +++ b/docs/tutorial.rst @@ -774,7 +774,7 @@ the following code:: Any other compatible storage class could be used in place of :class:`zarr.storage.DirectoryStore` in the code examples above. For example, -here is an array stored directly into a ZIP archive, via the +here is an array stored directly into a Zip file, via the :class:`zarr.storage.ZipStore` class:: >>> store = zarr.ZipStore('data/example.zip', mode='w') @@ -798,12 +798,12 @@ Re-open and check that data have been written:: [42, 42, 42, ..., 42, 42, 42]], dtype=int32) >>> store.close() -Note that there are some limitations on how ZIP archives can be used, because items -within a ZIP archive cannot be updated in place. This means that data in the array +Note that there are some limitations on how Zip files can be used, because items +within a Zip file cannot be updated in place. This means that data in the array should only be written once and write operations should be aligned with chunk boundaries. Note also that the ``close()`` method must be called after writing any data to the store, otherwise essential records will not be written to the -underlying ZIP archive. +underlying zip file. Another storage alternative is the :class:`zarr.storage.DBMStore` class, added in Zarr version 2.2. This class allows any DBM-style database to be used for @@ -846,7 +846,7 @@ respectively require the `redis-py `_ and `pymongo `_ packages to be installed. For compatibility with the `N5 `_ data format, Zarr also provides -an N5 backend (this is currently an experimental feature). Similar to the ZIP storage class, an +an N5 backend (this is currently an experimental feature). Similar to the zip storage class, an :class:`zarr.n5.N5Store` can be instantiated directly:: >>> store = zarr.N5Store('data/example.n5') @@ -868,7 +868,7 @@ implementations of the ``MutableMapping`` interface for Amazon S3 (`S3Map Distributed File System (`HDFSMap `_) and Google Cloud Storage (`GCSMap -`_), which +`_), which can be used with Zarr. Here is an example using S3Map to read an array created previously:: @@ -1000,32 +1000,6 @@ separately from Zarr. .. _tutorial_copy: -Accessing ZIP archives on S3 -~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -The built-in :class:`zarr.storage.ZipStore` will only work with paths on the local file-system; however -it is possible to access ZIP-archived Zarr data on the cloud via the `ZipFileSystem `_ -class from ``fsspec``. The following example demonstrates how to access -a ZIP-archived Zarr group on s3 using `s3fs `_ and ``ZipFileSystem``: - - >>> s3_path = "s3://path/to/my.zarr.zip" - >>> - >>> s3 = s3fs.S3FileSystem() - >>> f = s3.open(s3_path) - >>> fs = ZipFileSystem(f, mode="r") - >>> store = FSMap("", fs, check=False) - >>> - >>> # caching may improve performance when repeatedly reading the same data - >>> cache = zarr.storage.LRUStoreCache(store, max_size=2**28) - >>> z = zarr.group(store=cache) - -This store can also be generated with ``fsspec``'s handler chaining, like so: - - >>> store = zarr.storage.FSStore(url=f"zip::{s3_path}", mode="r") - -This can be especially useful if you have a very large ZIP-archived Zarr array or group on s3 -and only need to access a small portion of it. - Consolidating metadata ~~~~~~~~~~~~~~~~~~~~~~ @@ -1162,7 +1136,7 @@ re-compression, and so should be faster. E.g.:: └── spam (100,) int64 >>> new_root['foo/bar/baz'][:] array([ 0, 1, 2, ..., 97, 98, 99]) - >>> store2.close() # ZIP stores need to be closed + >>> store2.close() # zip stores need to be closed .. _tutorial_strings: diff --git a/pyproject.toml b/pyproject.toml index 2bc2b526e8..b77260b005 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -109,7 +109,6 @@ Homepage = "https://github.com/zarr-developers/zarr-python" exclude_lines = [ "pragma: no cover", "pragma: ${PY_MAJOR_VERSION} no cover", - '.*\.\.\.' # Ignore "..." lines ] [tool.coverage.run] diff --git a/v3-roadmap-and-design.md b/v3-roadmap-and-design.md deleted file mode 100644 index 696799e56f..0000000000 --- a/v3-roadmap-and-design.md +++ /dev/null @@ -1,429 +0,0 @@ -# Zarr Python Roadmap - -- Status: draft -- Author: Joe Hamman -- Created On: October 31, 2023 -- Input from: - - Davis Bennett / @d-v-b - - Norman Rzepka / @normanrz - - Deepak Cherian @dcherian - - Brian Davis / @monodeldiablo - - Oliver McCormack / @olimcc - - Ryan Abernathey / @rabernat - - Jack Kelly / @JackKelly - - Martin Durrant / @martindurant - -## Introduction - -This document lays out a design proposal for version 3.0 of the [Zarr-Python](https://zarr.readthedocs.io/en/stable/) package. A specific focus of the design is to bring Zarr-Python's API up to date with the [Zarr V3 specification](https://zarr-specs.readthedocs.io/en/latest/v3/core/v3.0.html), with the hope of enabling the development of the many features and extensions that motivated the V3 Spec. The ideas presented here are expected to result in a major release of Zarr-Python (version 3.0) including significant a number of breaking API changes. -For clarity, “V3” will be used to describe the version of the Zarr specification and “3.0” will be used to describe the release tag of the Zarr-Python project. - -### Current status of V3 in Zarr-Python - -During the development of the V3 Specification, a [prototype implementation](https://github.com/zarr-developers/zarr-python/pull/898) was added to the Zarr-Python library. Since that implementation, the V3 spec evolved in significant ways and as a result, the Zarr-Python library is now out of sync with the approved spec. Downstream libraries (e.g. [Xarray](https://github.com/pydata/xarray)) have added support for this implementation and will need to migrate to the accepted spec when its available in Zarr-Python. - -## Goals - -- Provide a complete implementation of Zarr V3 through the Zarr-Python API -- Clear the way for exciting extensions / ZEPs (i.e. [sharding](https://zarr-specs.readthedocs.io/en/latest/v3/codecs/sharding-indexed/v1.0.html), [variable chunking](https://zarr.dev/zeps/draft/ZEP0003.html), etc.) -- Provide a developer API that can be used to implement and register V3 extensions -- Improve the performance of Zarr-Python by streamlining the interface between the Store layer and higher level APIs (e.g. Groups and Arrays) -- Clean up the internal and user facing APIs -- Improve code quality and robustness (e.g. achieve 100% type hint coverage) -- Align the Zarr-Python array API with the [array API Standard](https://data-apis.org/array-api/latest/) - -## Examples of what 3.0 will enable? -1. Reading and writing V3 spec-compliant groups and arrays -2. V3 extensions including sharding and variable chunking. -3. Improved performance by leveraging concurrency when creating/reading/writing to stores (imagine a `create_hierarchy(zarr_objects)` function). -4. User-developed extensions (e.g. storage-transformers) can be registered with Zarr-Python at runtime - -## Non-goals (of this document) - -- Implementation of any unaccepted Zarr V3 extensions -- Major revisions to the Zarr V3 spec - -## Requirements - -1. Read and write spec compliant V2 and V3 data -2. Limit unnecessary traffic to/from the store -3. Cleanly define the Array/Group/Store abstractions -4. Cleanly define how V2 will be supported going forward -5. Provide a clear roadmap to help users upgrade to 3.0 -6. Developer tools / hooks for registering extensions - -## Design - -### Async API - -Zarr-Python is an IO library. As such, supporting concurrent action against the storage layer is critical to achieving acceptable performance. The Zarr-Python 2 was not designed with asynchronous computation in mind and as a result has struggled to effectively leverage the benefits of concurrency. At one point, `getitems` and `setitems` support was added to the Zarr store model but that is only used for operating on a set of chunks in a single variable. - -With Zarr-Python 3.0, we have the opportunity to revisit this design. The proposal here is as follows: - -1. The `Store` interface will be entirely async. -2. On top of the async `Store` interface, we will provide an `AsyncArray` and `AsyncGroup` interface. -3. Finally, the primary user facing API will be synchronous `Array` and `Group` classes that wrap the async equivalents. - -**Examples** - -- **Store** - - ```python - class Store: - ... - async def get(self, key: str) -> bytes: - ... - async def get_partial_values(self, key_ranges: List[Tuple[str, Tuple[int, Optional[int]]]]) -> bytes: - ... - # (no sync interface here) - ``` -- **Array** - - ```python - class AsyncArray: - ... - - async def getitem(self, selection: Selection) -> np.ndarray: - # the core logic for getitem goes here - - class Array: - _async_array: AsyncArray - - def __getitem__(self, selection: Selection) -> np.ndarray: - return sync(self._async_array.getitem(selection)) - ``` -- **Group** - - ```python - class AsyncGroup: - ... - - async def create_group(self, path: str, **kwargs) -> AsyncGroup: - # the core logic for create_group goes here - - class Group: - _async_group: AsyncGroup - - def create_group(self, path: str, **kwargs) -> Group: - return sync(self._async_group.create_group(path, **kwargs)) - ``` -**Internal Synchronization API** - -With the `Store` and core `AsyncArray`/ `AsyncGroup` classes being predominantly async, Zarr-Python will need an internal API to provide a synchronous API. The proposal here is to use the approach in [fsspec](https://github.com/fsspec/filesystem_spec/blob/master/fsspec/asyn.py) to provide a high-level `sync` function that takes an `awaitable` and runs it in its managed IO Loop / thread. - -**FAQ** -1. Why two levels of Arrays/groups? - a. First, this is an intentional decision and departure from the current Zarrita implementation - b. The idea is that users rarely want to mix interfaces. Either they are working within an async context (currently quite rare) or they are in a typical synchronous context. - c. Splitting the two will allow us to clearly define behavior on the `AsyncObj` and simply wrap it in the `SyncObj`. -2. What if a store is only has a synchronous backend? - a. First off, this is expected to be a fairly rare occurrence. Most storage backends have async interfaces. - b. But in the event a storage backend doesn’t have a async interface, there is nothing wrong with putting synchronous code in `async` methods. There are approaches to enabling concurrent action through wrappers like AsyncIO's `loop.run_in_executor` ([ref 1](https://stackoverflow.com/questions/38865050/is-await-in-python3-cooperative-multitasking ), [ref 2](https://stackoverflow.com/a/43263397/732596), [ref 3](https://bbc.github.io/cloudfit-public-docs/asyncio/asyncio-part-5.html), [ref 4](https://docs.python.org/3/library/asyncio-eventloop.html#asyncio.loop.run_in_executor). -3. Will Zarr help manage the async contexts encouraged by some libraries (e.g. [AioBotoCore](https://aiobotocore.readthedocs.io/en/latest/tutorial.html#using-botocore))? - a. Many async IO libraries require entering an async context before interacting with the API. We expect some experimentation to be needed here but the initial design will follow something close to what fsspec does ([example in s3fs](https://github.com/fsspec/s3fs/blob/949442693ec940b35cda3420c17a864fbe426567/s3fs/core.py#L527)). -4. Why not provide a synchronous Store interface? - a. We could but this design is simpler. It would mean supporting it in the `AsyncGroup` and `AsyncArray` classes which, may be more trouble than its worth. Storage backends that do not have an async API will be encouraged to wrap blocking calls in an async wrapper (e.g. `loop.run_in_executor`). - -### Store API - -The `Store` API is specified directly in the V3 specification. All V3 stores should implement this abstract API, omitting Write and List support as needed. As described above, all stores will be expected to expose the required methods as async methods. - -**Example** - -```python -class ReadWriteStore: - ... - async def get(self, key: str) -> bytes: - ... - - async def get_partial_values(self, key_ranges: List[Tuple[str, int, int]) -> bytes: - ... - - async def set(self, key: str, value: Union[bytes, bytearray, memoryview]) -> None: - ... # required for writable stores - - async def set_partial_values(self, key_start_values: List[Tuple[str, int, Union[bytes, bytearray, memoryview]]]) -> None: - ... # required for writable stores - - async def list(self) -> List[str]: - ... # required for listable stores - - async def list_prefix(self, prefix: str) -> List[str]: - ... # required for listable stores - - async def list_dir(self, prefix: str) -> List[str]: - ... # required for listable stores - - # additional (optional methods) - async def getsize(self, prefix: str) -> int: - ... - - async def rename(self, src: str, dest: str) -> None - ... - -``` - -Recognizing that there are many Zarr applications today that rely on the `MutableMapping` interface supported by Zarr-Python 2, a wrapper store will be developed to allow existing stores to plug directly into this API. - -### Array API - -The user facing array interface will implement a subset of the [Array API Standard](https://data-apis.org/array-api/latest/). Most of the computational parts of the Array API Standard don’t fit into Zarr right now. That’s okay. What matters most is that we ensure we can give downstream applications a compliant API. - -*Note, Zarr already does most of this so this is more about formalizing the relationship than a substantial change in API.* - -| | Included | Not Included | Unknown / Maybe possible? | -| --- | --- | --- | --- | -| Attributes | `dtype` | `mT` | `device` | -| | `ndim` | `T` | | -| | `shape` | | | -| | `size` | | | -| Methods | `__getitem__` | `__array_namespace__` | `to_device` | -| | `__setitem__` | `__abs__` | `__bool__` | -| | `__eq__` | `__add__` | `__complex__` | -| | `__bool__` | `__and__` | `__dlpack__` | -| | | `__floordiv__` | `__dlpack_device__` | -| | | `__ge__` | `__float__` | -| | | `__gt__` | `__index__` | -| | | `__invert__` | `__int__` | -| | | `__le__` | | -| | | `__lshift__` | | -| | | `__lt__` | | -| | | `__matmul__` | | -| | | `__mod__` | | -| | | `__mul__` | | -| | | `__ne__` | | -| | | `__neg__` | | -| | | `__or__` | | -| | | `__pos__` | | -| | | `__pow__` | | -| | | `__rshift__` | | -| | | `__sub__` | | -| | | `__truediv__` | | -| | | `__xor__` | | -| Creation functions (`zarr.creation`) | `zeros` | | `arange` | -| | `zeros_like` | | `asarray` | -| | `ones` | | `eye` | -| | `ones_like` | | `from_dlpack` | -| | `full` | | `linspace` | -| | `full_like` | | `meshgrid` | -| | `empty` | | `tril` | -| | `empty_like` | | `triu` | - -In addition to the core array API defined above, the Array class should have the following Zarr specific properties: - -- `.metadata` (see Metadata Interface below) -- `.attrs` - (pull from metadata object) -- `.info` - (pull from existing property †) - -*† In Zarr-Python 2, the info property lists the store to identify initialized chunks. By default this will be turned off in 3.0 but will be configurable.* - -**Indexing** - -Zarr-Python currently supports `__getitem__` style indexing and the special `oindex` and `vindex` indexers. These are not part of the current Array API standard (see [data-apis/array-api\#669](https://github.com/data-apis/array-api/issues/669)) but they have been [proposed as a NEP](https://numpy.org/neps/nep-0021-advanced-indexing.html). Zarr-Python will maintain these in 3.0. - -We are also exploring a new high-level indexing API that will enabled optimized batch/concurrent loading of many chunks. We expect this to be important to enable performant loading of data in the context of sharding. See [this discussion](https://github.com/zarr-developers/zarr-python/discussions/1569) for more detail. - -Concurrent indexing across multiple arrays will be possible using the AsyncArray API. - -**Async and Sync Array APIs** - -Most the logic to support Zarr Arrays will live in the `AsyncArray` class. There are a few notable differences that should be called out. - -| Sync Method | Async Method | -| --- | --- | -| `__getitem__` | `getitem` | -| `__setitem__` | `setitem` | -| `__eq__` | `equals` | - -**Metadata interface** - -Zarr-Python 2.* closely mirrors the V2 spec metadata schema in the Array and Group classes. In 3.0, we plan to move the underlying metadata representation to a separate interface (e.g. `Array.metadata`). This interface will return either a `V2ArrayMetadata` or `V3ArrayMetadata` object (both will inherit from a parent `ArrayMetadataABC` class. The `V2ArrayMetadata` and `V3ArrayMetadata` classes will be responsible for producing valid JSON representations of their metadata, and yielding a consistent view to the `Array` or `Group` class. - -### Group API - -The main question is how closely we should follow the existing Zarr-Python implementation / `MutableMapping` interface. The table below shows the primary `Group` methods in Zarr-Python 2 and attempts to identify if and how they would be implemented in 3.0. - -| V2 Group Methods | `AsyncGroup` | `Group` | `h5py_compat.Group`` | -| --- | --- | --- | --- | -| `__len__` | `length` | `__len__` | `__len__` | -| `__iter__` | `__aiter__` | `__iter__` | `__iter__` | -| `__contains__` | `contains` | `__contains__` | `__contains__` | -| `__getitem__` | `getitem` | `__getitem__` | `__getitem__` | -| `__enter__` | N/A | N/A | `__enter__` | -| `__exit__` | N/A | N/A | `__exit__` | -| `group_keys` | `group_keys` | `group_keys` | N/A | -| `groups` | `groups` | `groups` | N/A | -| `array_keys` | `array_key` | `array_keys` | N/A | -| `arrays` | `arrays`* | `arrays` | N/A | -| `visit` | ? | ? | `visit` | -| `visitkeys` | ? | ? | ? | -| `visitvalues` | ? | ? | ? | -| `visititems` | ? | ? | `visititems` | -| `tree` | `tree` | `tree` | `Both` | -| `create_group` | `create_group` | `create_group` | `create_group` | -| `require_group` | N/A | N/A | `require_group` | -| `create_groups` | ? | ? | N/A | -| `require_groups` | ? | ? | ? | -| `create_dataset` | N/A | N/A | `create_dataset` | -| `require_dataset` | N/A | N/A | `require_dataset` | -| `create` | `create_array` | `create_array` | N/A | -| `empty` | `empty` | `empty` | N/A | -| `zeros` | `zeros` | `zeros` | N/A | -| `ones` | `ones` | `ones` | N/A | -| `full` | `full` | `full` | N/A | -| `array` | `create_array` | `create_array` | N/A | -| `empty_like` | `empty_like` | `empty_like` | N/A | -| `zeros_like` | `zeros_like` | `zeros_like` | N/A | -| `ones_like` | `ones_like` | `ones_like` | N/A | -| `full_like` | `full_like` | `full_like` | N/A | -| `move` | `move` | `move` | `move` | - -**`zarr.h5compat.Group`** - -Zarr-Python 2.* made an attempt to align its API with that of [h5py](https://docs.h5py.org/en/stable/index.html). With 3.0, we will relax this alignment in favor of providing an explicit compatibility module (`zarr.h5py_compat`). This module will expose the `Group` and `Dataset` APIs that map to Zarr-Python’s `Group` and `Array` objects. - -### Creation API - -Zarr-Python 2.* bundles together the creation and serialization of Zarr objects. Zarr-Python 3.* will make it possible to create objects in memory separate from serializing them. This will specifically enable writing hierarchies of Zarr objects in a single batch step. For example: - -```python - -arr1 = Array(shape=(10, 10), path="foo/bar", dtype="i4", store=store) -arr2 = Array(shape=(10, 10), path="foo/spam", dtype="f8", store=store) - -arr1.save() -arr2.save() - -# or equivalently - -zarr.save_many([arr1 ,arr2]) -``` - -*Note: this batch creation API likely needs additional design effort prior to implementation.* - -### Plugin API - -Zarr V3 was designed to be extensible at multiple layers. Zarr-Python will support these extensions through a combination of [Abstract Base Classes](https://docs.python.org/3/library/abc.html) (ABCs) and [Entrypoints](https://packaging.python.org/en/latest/specifications/entry-points/). - -**ABCs** - -Zarr V3 will expose Abstract base classes for the following objects: - -- `Store`, `ReadStore`, `ReadWriteStore`, `ReadListStore`, and `ReadWriteListStore` -- `BaseArray`, `SynchronousArray`, and `AsynchronousArray` -- `BaseGroup`, `SynchronousGroup`, and `AsynchronousGroup` -- `Codec`, `ArrayArrayCodec`, `ArrayBytesCodec`, `BytesBytesCodec` - -**Entrypoints** - -Lots more thinking here but the idea here is to provide entrypoints for `data type`, `chunk grid`, `chunk key encoding`, `codecs`, `storage_transformers` and `stores`. These might look something like: - -``` -entry_points=""" - [zarr.codecs] - blosc_codec=codec_plugin:make_blosc_codec - zlib_codec=codec_plugin:make_zlib_codec -""" -``` - -### Python type hints and static analysis - -Target 100% Mypy coverage in 3.0 source. - -### Observability - -A persistent problem in Zarr-Python is diagnosing problems that span many parts of the stack. To address this in 3.0, we will add a basic logging framework that can be used to debug behavior at various levels of the stack. We propose to add the separate loggers for the following namespaces: - -- `array` -- `group` -- `store` -- `codec` - -These should be documented such that users know how to activate them and developers know how to use them when developing extensions. - -### Dependencies - -Today, Zarr-Python has the following required dependencies: - -```python -dependencies = [ - 'asciitree', - 'numpy>=1.20,!=1.21.0', - 'fasteners', - 'numcodecs>=0.10.0', -] -``` - -What other dependencies should be considered? - -1. Attrs - Zarrita makes extensive use of the Attrs library -2. Fsspec - Zarrita has a hard dependency on Fsspec. This could be easily relaxed though. - -## Breaking changes relative to Zarr-Python 2.* - -1. H5py compat moved to a stand alone module? -2. `Group.__getitem__` support moved to `Group.members.__getitem__`? -3. Others? - -## Open questions - -1. How to treat V2 - a. Note: Zarrita currently implements a separate `V2Array` and `V3Array` classes. This feels less than ideal. - b. We could easily convert metadata from v2 to the V3 Array, but what about writing? - c. Ideally, we don’t have completely separate code paths. But if its too complicated to support both within one interface, its probably better. -2. How and when to remove the current implementation of V3. - a. It's hidden behind a hard-to-use feature flag so we probably don't need to do anything. -4. How to model runtime configuration? -5. Which extensions belong in Zarr-Python and which belong in separate packages? - a. We don't need to take a strong position on this here. It's likely that someone will want to put Sharding in. That will be useful to develop in parallel because it will give us a good test case for the plugin interface. - -## Testing - -Zarr-python 3.0 adds a major new dimension to Zarr: Async support. This also comes with a compatibility risk, we will need to thoroughly test support in key execution environments. Testing plan: -- Reuse the existing test suite for testing the `v3` API. - - `xfail` tests that expose breaking changes with `3.0 - breaking change` description. This will help identify additional and/or unintentional breaking changes - - Rework tests that were only testing internal APIs. -- Add a set of functional / integration tests targeting real-world workflows in various contexts (e.g. w/ Dask) - -## Development process - -Zarr-Python 3.0 will introduce a number of new APIs and breaking changes to existing APIs. In order to facilitate ongoing support for Zarr-Python 2.*, we will take on the following development process: - -- Create a `v3` branch that can be use for developing the core functionality apart from the `main` branch. This will allow us to support ongoing work and bug fixes on the `main` branch. -- Put the `3.0` APIs inside a `zarr.v3` module. Imports from this namespace will all be new APIs that users can develop and test against once the `v3` branch is merged to `main`. -- Kickstart the process by pulling in the current state of `zarrita` - which has many of the features described in this design. -- Release a series of 2.* releases with the `v3` namespace -- When `v3` is complete, move contents of `v3` to the package root - -**Milestones** - -Below are a set of specific milestones leading toward the completion of this process. As work begins, we expect this list to grow in specificity. - -1. Port current version of Zarrita to Zarr-Python -2. Formalize Async interface by splitting `Array` and `Group` objects into Sync and Async versions -4. Implement "fancy" indexing operations on the `AsyncArray` -6. Implement an abstract base class for the `Store` interface and a wrapper `Store` to make use of existing `MutableMapping` stores. -7. Rework the existing unit test suite to use the `v3` namespace. -8. Develop a plugin interface for extensions -9. Develop a set of functional and integration tests -10. Work with downstream libraries (Xarray, Dask, etc.) to test new APIs - -## TODOs - -The following subjects are not covered in detail above but perhaps should be. Including them here so they are not forgotten. - -1. [Store] Should Zarr provide an API for caching objects after first read/list/etc. Read only stores? -2. [Array] buffer protocol support -3. [Array] `meta_array` support -4. [Extensions] Define how Zarr-Python will consume the various plugin types -5. [Misc] H5py compatibility requires a bit more work and a champion to drive it forward. -6. [Misc] Define `chunk_store` API in 3.0 -7. [Misc] Define `synchronizer` API in 3.0 - -## References - -1. [Zarr-Python repository](https://github.com/zarr-developers/zarr-python) -2. [Zarr core specification (version 3.0) — Zarr specs documentation](https://zarr-specs.readthedocs.io/en/latest/v3/core/v3.0.html#) -3. [Zarrita repository](https://github.com/scalableminds/zarrita) -4. [Async-Zarr](https://github.com/martindurant/async-zarr) -5. [Zarr-Python Discussion Topic](https://github.com/zarr-developers/zarr-python/discussions/1569)