Merge branch 'main' into format_specification

.github/workflows/minimal.yml

@@ -15,7 +15,7 @@ jobs:
     steps:
     - uses: actions/checkout@v4
     - name: Setup Miniconda
-      uses: conda-incubator/setup-miniconda@v2.2.0
+      uses: conda-incubator/setup-miniconda@v3.0.1
       with:
           channels: conda-forge
           environment-file: environment.yml

.github/workflows/python-package.yml

@@ -42,7 +42,7 @@ jobs:
       with:
         fetch-depth: 0
     - name: Setup Miniconda
-      uses: conda-incubator/setup-miniconda@v2.2.0
+      uses: conda-incubator/setup-miniconda@v3.0.1
       with:
           channels: conda-forge
           python-version: ${{ matrix.python-version }}

.github/workflows/releases.yml

@@ -64,7 +64,7 @@ jobs:
         with:
           name: releases
           path: dist
-      - uses: pypa/gh-action-pypi-publish@v1.8.10
+      - uses: pypa/gh-action-pypi-publish@v1.8.11
         with:
           user: __token__
           password: ${{ secrets.pypi_password }}

.github/workflows/windows-testing.yml

@@ -21,7 +21,7 @@ jobs:
       - uses: actions/checkout@v4
         with:
           fetch-depth: 0
-      - uses: conda-incubator/setup-miniconda@v2.2.0
+      - uses: conda-incubator/setup-miniconda@v3.0.1
         with:
           auto-update-conda: true
           python-version: ${{ matrix.python-version }}

.gitignore

@@ -51,6 +51,7 @@ coverage.xml
 
 # Sphinx documentation
 docs/_build/
+docs/_autoapi/
 
 # PyBuilder
 target/

.pre-commit-config.yaml

@@ -14,7 +14,7 @@ repos:
       # Respect `exclude` and `extend-exclude` settings.
       args: ["--force-exclude"]
   - repo: https://github.com/psf/black
-    rev: 22.12.0
+    rev: 23.10.1
     hooks:
     - id: black
   - repo: https://github.com/codespell-project/codespell

.readthedocs.yaml

@@ -7,10 +7,13 @@ build:
 
 sphinx:
   configuration: docs/conf.py
+  fail_on_warning: true
 
 python:
   install:
     - method: pip
       path: .
       extra_requirements:
         - docs
+
+formats: all

bench/compress_normal.py

@@ -8,7 +8,6 @@
 from zarr import blosc
 
 if __name__ == "__main__":
-
     sys.path.insert(0, "..")
 
     # setup

docs/Makefile

@@ -2,7 +2,7 @@
 #
 
 # You can set these variables from the command line.
-SPHINXOPTS    =
+SPHINXOPTS    = -W --keep-going
 SPHINXBUILD   = sphinx-build
 PAPER         =
 BUILDDIR      = _build

docs/api/core.rst

@@ -1,24 +1,5 @@
 The Array class (``zarr.core``)
 ===============================
-.. module:: zarr.core
 
-.. autoclass:: Array
-
-    .. automethod:: __getitem__
-    .. automethod:: __setitem__
-    .. automethod:: get_basic_selection
-    .. automethod:: set_basic_selection
-    .. automethod:: get_mask_selection
-    .. automethod:: set_mask_selection
-    .. automethod:: get_block_selection
-    .. automethod:: set_block_selection
-    .. automethod:: get_coordinate_selection
-    .. automethod:: set_coordinate_selection
-    .. automethod:: get_orthogonal_selection
-    .. automethod:: set_orthogonal_selection
-    .. automethod:: digest
-    .. automethod:: hexdigest
-    .. automethod:: resize
-    .. automethod:: append
-    .. automethod:: view
-    .. automethod:: astype
+.. automodapi:: zarr.core
+   :no-heading:

docs/conf.py

@@ -42,6 +42,7 @@
     "sphinx.ext.autosummary",
     "sphinx.ext.viewcode",
     "sphinx.ext.intersphinx",
+    "sphinx_automodapi.automodapi",
     "numpydoc",
     "sphinx_issues",
     "sphinx_copybutton",
@@ -52,6 +53,9 @@
 numpydoc_class_members_toctree = False
 issues_github_path = "zarr-developers/zarr-python"
 
+automodapi_inheritance_diagram = False
+automodapi_toctreedirnm = "_autoapi"
+
 # Add any paths that contain templates here, relative to this directory.
 templates_path = ["_templates"]
 
@@ -144,6 +148,7 @@
         },
     ],
     "collapse_navigation": True,
+    "navigation_with_keys": False,
 }
 
 # Add any paths that contain custom themes here, relative to this directory.
@@ -331,6 +336,7 @@ def setup(app):
 intersphinx_mapping = {
     "python": ("https://docs.python.org/", None),
     "numpy": ("https://numpy.org/doc/stable/", None),
+    "numcodecs": ("https://numcodecs.readthedocs.io/en/stable/", None),
 }
 
 

docs/contributing.rst

@@ -85,9 +85,9 @@ Creating a development environment
 To work with the Zarr source code, it is recommended to set up a Python virtual
 environment and install all Zarr dependencies using the same versions as are used by
 the core developers and continuous integration services. Assuming you have a Python
-3 interpreter already installed, and have also installed the virtualenv package, and
-you have cloned the Zarr source code and your current working directory is the root of
-the repository, you can do something like the following::
+3 interpreter already installed, and you have cloned the Zarr source code and your
+current working directory is the root of the repository, you can do something like
+the following::
 
     $ mkdir -p ~/pyenv/zarr-dev
     $ python -m venv ~/pyenv/zarr-dev

docs/index.rst

@@ -19,7 +19,7 @@ Zarr-Python
 
 **Version**: |version|
 
-**Download documentation**: `Zipped HTML <https://zarr.readthedocs.io/_/downloads/en/stable/htmlzip/>`_
+**Download documentation**: `PDF/Zipped HTML/EPUB <https://readthedocs.org/projects/zarr/downloads/>`_
 
 **Useful links**:
 `Installation <installation.html>`_ |
@@ -60,6 +60,7 @@ Zarr is a file storage format for chunked, compressed, N-dimensional arrays base
         +++
 
         .. button-ref:: tutorial
+            :ref-type: ref
             :expand:
             :color: dark
             :click-parent:

docs/release.rst

@@ -18,9 +18,37 @@ Release notes
 Unreleased
 ----------
 
+Docs
+~~~~
+
+* Minor correction and changes in documentation.
+  By :user:`Sanket Verma <MSanKeys963>` :issue:`1509`.
+
+* Fix typo in documentation.
+  By :user:`Dimitri Papadopoulos Orfanos <DimitriPapadopoulos>` :issue:`1554`
+
+* The documentation build now fails if there are any warnings.
+  By :user:`David Stansby <dstansby>` :issue:`1548`.
+
+* Add links to ``numcodecs`` docs in the tutorial.
+  By :user:`David Stansby <dstansby>` :issue:`1535`.
+
+* Enable offline formats for documentation builds.
+  By :user:`Sanket Verma <MSanKeys963>` :issue:`1551`.
+
+* Minor tweak to advanced indexing tutorial examples.
+  By :user:`Ross Barnowski <rossbar>` :issue:`1550`.
+
+
 Maintenance
 ~~~~~~~~~~~
 
+* Cache result of ``FSStore._fsspec_installed()``.
+  By :user:`Janick Martinez Esturo <ph03>` :issue:`1581`.
+
+* Extend copyright notice to 2023.
+  By :user:`Jack Kelly <JackKelly>` :issue:`1528`.
+
 * Change occurrence of ``io.open()`` into ``open()``.
   By :user:`Dimitri Papadopoulos Orfanos <DimitriPapadopoulos>` :issue:`1421`.
 
@@ -33,6 +61,10 @@ Maintenance
 * Allow ``black`` code formatter to be run with any Python version.
   By :user:`David Stansby <dstansby>` :issue:`1549`.
 
+* Remove ``sphinx-rtd-theme`` dependency from ``pyproject.toml``.
+  By :user:`Sanket Verma <MSanKeys963>` :issue:`1563`.
+
+
 .. _release_2.16.1:
 
 2.16.1
@@ -161,10 +193,12 @@ Major changes
 
 * Improve Zarr V3 support, adding partial store read/write and storage transformers.
   Add new features from the `v3 spec <https://zarr-specs.readthedocs.io/en/latest/core/v3.0.html>`_:
+
     * storage transformers
     * `get_partial_values` and `set_partial_values`
     * efficient `get_partial_values` implementation for `FSStoreV3`
     * sharding storage transformer
+
   By :user:`Jonathan Striebel <jstriebel>`; :issue:`1096`, :issue:`1111`.
 
 * N5 nows supports Blosc.

docs/tutorial.rst

@@ -480,17 +480,17 @@ Indexing with coordinate arrays
 Items from a Zarr array can be extracted by providing an integer array of
 coordinates. E.g.::
 
-    >>> z = zarr.array(np.arange(10))
+    >>> z = zarr.array(np.arange(10) ** 2)
     >>> z[:]
-    array([0, 1, 2, 3, 4, 5, 6, 7, 8, 9])
-    >>> z.get_coordinate_selection([1, 4])
-    array([1, 4])
+    array([ 0,  1,  4,  9, 16, 25, 36, 49, 64, 81])
+    >>> z.get_coordinate_selection([2, 5])
+    array([ 4, 25])
 
 Coordinate arrays can also be used to update data, e.g.::
 
-    >>> z.set_coordinate_selection([1, 4], [-1, -2])
+    >>> z.set_coordinate_selection([2, 5], [-1, -2])
     >>> z[:]
-    array([ 0, -1,  2,  3, -2,  5,  6,  7,  8,  9])
+    array([ 0,  1, -1,  9, 16, -2, 36, 49, 64, 81])
 
 For multidimensional arrays, coordinates must be provided for each dimension,
 e.g.::
@@ -534,17 +534,17 @@ Indexing with a mask array
 
 Items can also be extracted by providing a Boolean mask. E.g.::
 
-    >>> z = zarr.array(np.arange(10))
+    >>> z = zarr.array(np.arange(10) ** 2)
     >>> z[:]
-    array([0, 1, 2, 3, 4, 5, 6, 7, 8, 9])
+    array([ 0,  1,  4,  9, 16, 25, 36, 49, 64, 81])
     >>> sel = np.zeros_like(z, dtype=bool)
-    >>> sel[1] = True
-    >>> sel[4] = True
+    >>> sel[2] = True
+    >>> sel[5] = True
     >>> z.get_mask_selection(sel)
-    array([1, 4])
+    array([ 4, 25])
     >>> z.set_mask_selection(sel, [-1, -2])
     >>> z[:]
-    array([ 0, -1,  2,  3, -2,  5,  6,  7,  8,  9])
+    array([ 0,  1, -1,  9, 16, -2, 36, 49, 64, 81])
 
 Here's a multidimensional example::
 
@@ -986,7 +986,7 @@ It is also possible to initialize the filesystem outside of Zarr and then pass
 it through. This requires creating an :class:`zarr.storage.FSStore` object
 explicitly. For example::
 
-    >>> import s3fs  * doctest: +SKIP
+    >>> import s3fs  # doctest: +SKIP
     >>> fs = s3fs.S3FileSystem(anon=True)  # doctest: +SKIP
     >>> store = zarr.storage.FSStore('/zarr-demo/store', fs=fs)  # doctest: +SKIP
     >>> g = zarr.open_group(store)  # doctest: +SKIP
@@ -1175,8 +1175,9 @@ A fixed-length unicode dtype is also available, e.g.::
 For variable-length strings, the ``object`` dtype can be used, but a codec must be
 provided to encode the data (see also :ref:`tutorial_objects` below). At the time of
 writing there are four codecs available that can encode variable length string
-objects: :class:`numcodecs.VLenUTF8`, :class:`numcodecs.JSON`, :class:`numcodecs.MsgPack`.
-and :class:`numcodecs.Pickle`. E.g. using ``VLenUTF8``::
+objects: :class:`numcodecs.vlen.VLenUTF8`, :class:`numcodecs.json.JSON`,
+:class:`numcodecs.msgpacks.MsgPack`. and :class:`numcodecs.pickles.Pickle`.
+E.g. using ``VLenUTF8``::
 
     >>> import numcodecs
     >>> z = zarr.array(text_data, dtype=object, object_codec=numcodecs.VLenUTF8())
@@ -1201,8 +1202,8 @@ is a short-hand for ``dtype=object, object_codec=numcodecs.VLenUTF8()``, e.g.::
            'Helló, világ!', 'Zdravo svete!', 'เฮลโลเวิลด์'], dtype=object)
 
 Variable-length byte strings are also supported via ``dtype=object``. Again an
-``object_codec`` is required, which can be one of :class:`numcodecs.VLenBytes` or
-:class:`numcodecs.Pickle`. For convenience, ``dtype=bytes`` (or ``dtype=str`` on Python
+``object_codec`` is required, which can be one of :class:`numcodecs.vlen.VLenBytes` or
+:class:`numcodecs.pickles.Pickle`. For convenience, ``dtype=bytes`` (or ``dtype=str`` on Python
 2.7) can be used as a short-hand for ``dtype=object, object_codec=numcodecs.VLenBytes()``,
 e.g.::
 
@@ -1218,7 +1219,7 @@ e.g.::
            b'\xe0\xb9\x80\xe0\xb8\xae\xe0\xb8\xa5\xe0\xb9\x82\xe0\xb8\xa5\xe0\xb9\x80\xe0\xb8\xa7\xe0\xb8\xb4\xe0\xb8\xa5\xe0\xb8\x94\xe0\xb9\x8c'], dtype=object)
 
 If you know ahead of time all the possible string values that can occur, you could
-also use the :class:`numcodecs.Categorize` codec to encode each unique string value as an
+also use the :class:`numcodecs.categorize.Categorize` codec to encode each unique string value as an
 integer. E.g.::
 
     >>> categorize = numcodecs.Categorize(greetings, dtype=object)
@@ -1245,7 +1246,7 @@ The best codec to use will depend on what type of objects are present in the arr
 
 At the time of writing there are three codecs available that can serve as a general
 purpose object codec and support encoding of a mixture of object types:
-:class:`numcodecs.JSON`, :class:`numcodecs.MsgPack`. and :class:`numcodecs.Pickle`.
+:class:`numcodecs.json.JSON`, :class:`numcodecs.msgpacks.MsgPack`. and :class:`numcodecs.pickles.Pickle`.
 
 For example, using the JSON codec::
 
@@ -1258,7 +1259,7 @@ For example, using the JSON codec::
     array([42, 'foo', list(['bar', 'baz', 'qux']), {'a': 1, 'b': 2.2}, None], dtype=object)
 
 Not all codecs support encoding of all object types. The
-:class:`numcodecs.Pickle` codec is the most flexible, supporting encoding any type
+:class:`numcodecs.pickles.Pickle` codec is the most flexible, supporting encoding any type
 of Python object. However, if you are sharing data with anyone other than yourself, then
 Pickle is not recommended as it is a potential security risk. This is because malicious
 code can be embedded within pickled data. The JSON and MsgPack codecs do not have any
@@ -1270,7 +1271,7 @@ Ragged arrays
 
 If you need to store an array of arrays, where each member array can be of any length
 and stores the same primitive type (a.k.a. a ragged array), the
-:class:`numcodecs.VLenArray` codec can be used, e.g.::
+:class:`numcodecs.vlen.VLenArray` codec can be used, e.g.::
 
     >>> z = zarr.empty(4, dtype=object, object_codec=numcodecs.VLenArray(int))
     >>> z