Add support for patching, merging, replacing metadata (bluesky#688)

* add support to patch, merge, replace metadata

* patch_metadata uses a http patch request with
  application/json-patch+json content type
* merge_metadata can be triggered with a http
  patch request and application/merge-patch+json
* add jsonpatch and json-merge-patch to requirements

* lint suggestions

* fix compat with python<3.11 by removing `NotRequired`

* lint

* fix compat with python 3.8 by removing `Required`

* IMP `dict.update`-like `update_metadata()` method

*) client-side construction of a json patch using input similar to
dict.update
*) special value `tiled.client.metadata_update.DELETE_KEY` for deleting
keys (similar to `None` in json merge patch, but regains `None` as a valid value)

* review suggestions 1/2

* Use `Sentinel` for `DELETE_KEY`
* factor out `validate_metadata` utility function

* rearranged `validate_metadata` to make diff readable

* use SQL-backed adapter

* lint

* hopeful fix for CI error in py<3.10 🤞

* will `typing_extensions.TypedDict` make CI overlords happy?

* Apply suggestions from code review

Co-authored-by: Dan Allan <daniel.b.allan@gmail.com>

* Implement CR suggestions

* FEAT: option to retrieve JSON patch object without applying it

* fix typo

* reword examples in `update_metadata`

* Add CR suggestions from @danielballan

* add return type of `build_metadata_patch` to docstring

* Update uniqueness check to current best practice.

* 📖 Add `application/json-patch+json` dropdown option to openAPI docs

* implement sub-mimetype in patch payload

* linter suggestions

* some pydantic2 migration fixes

* pin python<3.11.9 to avoid dask error

* remove python pin added in c140311

* restrict to standard mimetype ids, improve error message

* add metadata patching in CHANGELOG

* fix typo

* add pytests for json- and merge- patching of metadata and specs

* refactor patch mimetypes

* add support for specs patching

* Remove unused import introduced by rebase.

* Update API docs for new name.

* Fix docstring formatting.

---------

Co-authored-by: Dan Allan <daniel.b.allan@gmail.com>
Co-authored-by: Dan Allan <dallan@bnl.gov>
Co-authored-by: kari Barry <kezzsim@gmail.com>

CHANGELOG.md

@@ -5,14 +5,24 @@ Write the date in place of the "Unreleased" in the case a new version is release
 
 ## Unreleased
 
-- Fix regression introduced in the previous release (v0.1.0b1) where exceptions
+### Added
+
+- Added a new HTTP endpoint, `PATCH /api/v1/metadata/{path}` supporting modifying
+  existing metadata using a `application/json-patch+json` or a
+  `application/merge-patch+json` patch.
+- Added client-side methods for replacing, updating (similar to `dict.update()`),
+  and patching metadata.
+
+### Fixed
+
+- Fixed regression introduced in the previous release (v0.1.0b1) where exceptions
   raised in the server sent _no_ response instead of properly sending a 500
   response. (This presents in the client as, "Server disconnected without
   sending a response.") A test now protects against this class of regression.
 
 ## v0.1.0b2 (2024-05-28)
 
-## Changed
+### Changed
 
 - Customized default logging configuration to include correlation ID and username
   of authenticated user.

docs/source/reference/python-client.md

@@ -137,6 +137,11 @@ Tiled currently includes two clients for each structure family:
    tiled.client.base.BaseClient
    tiled.client.base.BaseClient.formats
    tiled.client.base.BaseClient.metadata
+   tiled.client.base.BaseClient.metadata_copy
+   tiled.client.base.BaseClient.replace_metadata
+   tiled.client.base.BaseClient.update_metadata
+   tiled.client.base.BaseClient.patch_metadata
+   tiled.client.base.BaseClient.build_metadata_patches
    tiled.client.base.BaseClient.uri
    tiled.client.base.BaseClient.structure_family
    tiled.client.base.BaseClient.item

pyproject.toml

@@ -66,6 +66,8 @@ all = [
     "jinja2",
     "jmespath",
     "jsonschema",
+    "jsonpatch",
+    "json-merge-patch",
     "lz4",
     "msgpack >=1.0.0",
     "ndindex",
@@ -110,6 +112,7 @@ client = [
     "entrypoints",
     "httpx >=0.20.0,!=0.23.1",
     "jsonschema",
+    "jsonpatch",
     "lz4",
     "msgpack >=1.0.0",
     "ndindex",
@@ -238,6 +241,8 @@ server = [
     "jinja2",
     "jmespath",
     "jsonschema",
+    "jsonpatch",
+    "json-merge-patch",
     "lz4",
     "msgpack >=1.0.0",
     "ndindex",

tiled/_tests/test_writing.py

@@ -30,6 +30,7 @@
 from ..structures.data_source import DataSource
 from ..structures.sparse import COOStructure
 from ..structures.table import TableStructure
+from ..utils import patch_mimetypes
 from ..validation_registration import ValidationRegistry
 from .utils import fail_with_status_code
 
@@ -314,6 +315,51 @@ def test_metadata_revisions(tree):
             ac.metadata_revisions.delete_revision(1)
 
 
+def test_merge_patching(tree):
+    "Test merge patching of metadata and specs"
+    with Context.from_app(build_app(tree)) as context:
+        client = from_context(context)
+        ac = client.write_array([1, 2, 3], metadata={"a": 0, "b": 2}, specs=["spec1"])
+        ac.patch_metadata(
+            metadata_patch={"a": 1, "c": 3}, content_type=patch_mimetypes.MERGE_PATCH
+        )
+        assert dict(ac.metadata) == {"a": 1, "b": 2, "c": 3}
+        assert ac.specs[0].name == "spec1"
+        ac.patch_metadata(
+            specs_patch=["spec2"], content_type=patch_mimetypes.MERGE_PATCH
+        )
+        assert [x.name for x in ac.specs] == ["spec2"]
+
+
+def test_json_patching(tree):
+    "Test json patching of metadata and specs"
+
+    validation_registry = ValidationRegistry()
+
+    for i in range(10):
+        validation_registry.register(f"spec{i}", lambda *args, **kwargs: None)
+
+    with Context.from_app(
+        build_app(tree, validation_registry=validation_registry)
+    ) as context:
+        client = from_context(context)
+        ac = client.write_array([1, 2, 3], metadata={"a": 0, "b": 2}, specs=["spec1"])
+        ac.patch_metadata(
+            metadata_patch=[
+                {"op": "add", "path": "/c", "value": 3},
+                {"op": "replace", "path": "/a", "value": 1},
+            ],
+            content_type=patch_mimetypes.JSON_PATCH,
+        )
+        assert dict(ac.metadata) == {"a": 1, "b": 2, "c": 3}
+        assert ac.specs[0].name == "spec1"
+        ac.patch_metadata(
+            specs_patch=[{"op": "add", "path": "/1", "value": "spec2"}],
+            content_type=patch_mimetypes.JSON_PATCH,
+        )
+        assert [x.name for x in ac.specs] == ["spec1", "spec2"]
+
+
 def test_metadata_with_unsafe_objects(tree):
     with Context.from_app(build_app(tree)) as context:
         client = from_context(context)

tiled/catalog/adapter.py

@@ -916,7 +916,7 @@ async def delete_revision(self, number):
             ), f"Deletion would affect {result.rowcount} rows; rolling back"
             await db.commit()
 
-    async def update_metadata(self, metadata=None, specs=None):
+    async def replace_metadata(self, metadata=None, specs=None):
         values = {}
         if metadata is not None:
             # Trailing underscore in 'metadata_' avoids collision with

tiled/client/__init__.py

@@ -3,3 +3,4 @@
 from .container import ASCENDING, DESCENDING  # noqa: F401
 from .context import Context  # noqa: F401
 from .logger import hide_logs, record_history, show_logs  # noqa: F401
+from .metadata_update import DELETE_KEY  # noqa: F401