make consolidated metadata human-readable

Author: alimanfoo

zarr/attrs.py

@@ -4,8 +4,8 @@
 from collections import MutableMapping
 
 
-from zarr.compat import text_type
 from zarr.errors import PermissionError
+from zarr.meta import parse_metadata
 
 
 class Attributes(MutableMapping):
@@ -43,7 +43,7 @@ def _get_nosync(self):
         except KeyError:
             d = dict()
         else:
-            d = json.loads(text_type(data, 'ascii'))
+            d = parse_metadata(data)
         return d
 
     def asdict(self):

zarr/compat.py

@@ -19,6 +19,8 @@ class PermissionError(Exception):
     def OrderedDict_move_to_end(od, key):
         od[key] = od.pop(key)
 
+    from collections import Mapping
+
 
 else:  # pragma: py2 no cover
 
@@ -29,3 +31,5 @@ def OrderedDict_move_to_end(od, key):
 
     def OrderedDict_move_to_end(od, key):
         od.move_to_end(key)
+
+    from collections.abc import Mapping

zarr/convenience.py

@@ -15,6 +15,7 @@
 from zarr.errors import err_path_not_found, CopyError
 from zarr.util import normalize_storage_path, TreeViewer, buffer_size
 from zarr.compat import PY2, text_type
+from zarr.meta import ensure_str, json_dumps
 
 
 # noinspection PyShadowingBuiltins
@@ -1119,8 +1120,15 @@ def is_zarr_key(key):
         return (key.endswith('.zarray') or key.endswith('.zgroup') or
                 key.endswith('.zattrs'))
 
-    out = {key: store[key].decode() for key in store if is_zarr_key(key)}
-    store[metadata_key] = json.dumps(out).encode()
+#    out = {key: store[key].decode() for key in store if is_zarr_key(key)}
+    out = {
+        'zarr_consolidated_format': 1,
+        'metadata': {
+            key: json.loads(ensure_str(store[key]))
+            for key in store if is_zarr_key(key)
+        }
+    }
+    store[metadata_key] = json_dumps(out).encode()
     return open_consolidated(store, metadata_key=metadata_key)
 
 

zarr/core.py

@@ -165,6 +165,9 @@ def _load_metadata_nosync(self):
             if config is None:
                 self._compressor = None
             else:
+                # temporary workaround for
+                # https://github.com/zarr-developers/numcodecs/issues/78
+                config = dict(config)
                 self._compressor = get_codec(config)
 
             # setup filters

zarr/meta.py

@@ -7,14 +7,14 @@
 import numpy as np
 
 
-from zarr.compat import PY2, binary_type
+from zarr.compat import PY2, binary_type, Mapping
 from zarr.errors import MetadataError
 
 
 ZARR_FORMAT = 2
 
 
-def _ensure_str(s):
+def ensure_str(s):
     if PY2:  # pragma: py3 no cover
         # noinspection PyUnresolvedReferences
         if isinstance(s, buffer):  # noqa
@@ -27,12 +27,32 @@ def _ensure_str(s):
     return s
 
 
+def json_dumps(o):
+    """Write JSON in a consistent, human-readable way."""
+    return json.dumps(o, indent=4, sort_keys=True, ensure_ascii=True,
+                      separators=(',', ': '))
+
+
+def parse_metadata(s):
+    if isinstance(s, Mapping):
+        # assume metadata has already been parsed into a mapping object
+        meta = s
+    else:
+        # assume metadata needs to be parsed as JSON
+        s = ensure_str(s)
+        meta = json.loads(s)
+    return meta
+
+
 def decode_array_metadata(s):
-    s = _ensure_str(s)
-    meta = json.loads(s)
+    meta = parse_metadata(s)
+
+    # check metadata format
     zarr_format = meta.get('zarr_format', None)
     if zarr_format != ZARR_FORMAT:
         raise MetadataError('unsupported zarr format: %s' % zarr_format)
+
+    # extract array metadata fields
     try:
         dtype = decode_dtype(meta['dtype'])
         fill_value = decode_fill_value(meta['fill_value'], dtype)
@@ -67,8 +87,7 @@ def encode_array_metadata(meta):
         order=meta['order'],
         filters=meta['filters'],
     )
-    s = json.dumps(meta, indent=4, sort_keys=True, ensure_ascii=True,
-                   separators=(',', ': '))
+    s = json_dumps(meta)
     b = s.encode('ascii')
     return b
 
@@ -98,14 +117,14 @@ def decode_dtype(d):
 
 
 def decode_group_metadata(s):
-    s = _ensure_str(s)
-    meta = json.loads(s)
+    meta = parse_metadata(s)
+
+    # check metadata format version
     zarr_format = meta.get('zarr_format', None)
     if zarr_format != ZARR_FORMAT:
         raise MetadataError('unsupported zarr format: %s' % zarr_format)
-    meta = dict(
-        zarr_format=ZARR_FORMAT,
-    )
+
+    meta = dict(zarr_format=zarr_format)
     return meta
 
 
@@ -115,7 +134,7 @@ def encode_group_metadata(meta=None):
     meta = dict(
         zarr_format=ZARR_FORMAT,
     )
-    s = json.dumps(meta, indent=4, sort_keys=True, ensure_ascii=True)
+    s = json_dumps(meta)
     b = s.encode('ascii')
     return b
 

zarr/storage.py

@@ -41,7 +41,7 @@
 from zarr.compat import PY2, binary_type, OrderedDict_move_to_end
 from numcodecs.registry import codec_registry
 from zarr.errors import (err_contains_group, err_contains_array, err_bad_compressor,
-                         err_fspath_exists_notdir, err_read_only)
+                         err_fspath_exists_notdir, err_read_only, MetadataError)
 
 
 array_meta_key = '.zarray'
@@ -1932,12 +1932,22 @@ class ConsolidatedMetadataStore(MutableMapping):
     """
     def __init__(self, store, metadata_key='.zmetadata'):
         self.store = store
+
+        # retrieve consolidated metadata
         if sys.version_info.major == 3 and sys.version_info.minor < 6:
             d = store[metadata_key].decode()  # pragma: no cover
         else:  # pragma: no cover
             d = store[metadata_key]
-        metadata = json.loads(d)
-        self.meta_store = {k: v.encode() for k, v in metadata.items()}
+        meta = json.loads(d)
+
+        # check format of consolidated metadata
+        consolidated_format = meta.get('zarr_consolidated_format', None)
+        if consolidated_format != 1:
+            raise MetadataError('unsupported zarr consolidated metadata format: %s' %
+                                consolidated_format)
+
+        # decode metadata
+        self.meta_store = meta['metadata']
 
     def __getitem__(self, key):
         return self.meta_store[key]