Little framework for customizing the handling of low-level objects

.gitignore

@@ -4,3 +4,4 @@ develop-eggs
 dist
 doc/_build
 eggs
+.DS_Store

CHANGES.rst

@@ -4,7 +4,11 @@ Changes
 0.4.1 (unreleased)
 ==================
 
-- Nothing changed yet.
+- Added another data-transformation option:
+
+  reducer (advanced)
+    Callable called during JSON conversion to control how internal
+    (non-persistent) objects are converted to JSON.
 
 
 0.4.0 (2017-03-25)

doc/reference.rst

@@ -47,7 +47,7 @@ newt.db.jsonpickle module-level functions
 =========================================
 
 .. automodule:: newt.db.jsonpickle
-   :members: JsonUnpickler, Jsonifier
+   :members: JsonUnpickler, Jsonifier, dumps
 
    .. autoclass:: newt.db.jsonpickle.Jsonifier
       :members: __init__, __call__

doc/topics/data-transformation.rst

@@ -1,3 +1,4 @@
+====================
 Data transformation
 ====================
 
@@ -98,3 +99,48 @@ option to supply the dotted name of your transform function in the
    True
 
    >>> db.close()
+
+Low-level transformation: reducers
+==================================
+
+Transforms operate at the record level, after a database record's
+contents have been converted to JSON. There's also an **advanced** hook to
+supply limited transformation during the process of converting to JSON.
+
+See :py:class:`~newt.db.jsonpickle.JsonUnpickler` for details.
+
+You can supply the dotted name of a reducer in your configuration:
+
+.. code-block:: xml
+
+  %import newt.db
+
+  <newtdb>
+    <zodb>
+      <relstorage>
+        keep-history false
+        <newt>
+          transform myproject.flatten_persistent_mapping
+          reducer myproject.fancypants_reducer
+          <postgresql>
+            dsn dbname=''
+          </postgresql>
+        </newt>
+      </relstorage>
+    </zodb>
+  </newtdb>
+
+.. -> src
+
+   >>> from newt.db.tests import testdocs
+   >>> testdocs.flatten_persistent_mapping = flatten_persistent_mapping
+   >>> testdocs.fancypants_reducer = lambda *args: None
+   >>> src = src.replace('myproject', 'newt.db.tests.testdocs')
+
+   >>> src = src.replace("''", dsn.rsplit('/')[-1])
+   >>> from ZODB.config import databaseFromString
+   >>> db = databaseFromString(src)
+   >>> (db.storage._adapter.mover.jsonifier.reducer is
+   ...  testdocs.fancypants_reducer)
+   True
+   >>> db.close()

src/newt/db/_adapter.py

@@ -27,7 +27,9 @@ def __init__(self, *args, **kw):
         self.connmanager.set_on_store_opened(self.mover.on_store_opened)
 
         self.mover.jsonifier = Jsonifier(
-            transform=getattr(self.options, 'transform', None))
+            transform=getattr(self.options, 'transform', None),
+            reducer=getattr(self.options, 'reducer', None)
+            )
 
 class Mover(relstorage.adapters.postgresql.mover.PostgreSQLObjectMover):
 

src/newt/db/_db.py

@@ -108,7 +108,7 @@ def _split_options(
         large_record_size=large_record_size,
         ), storage_options
 
-def storage(dsn, keep_history=False, transform=None, **kw):
+def storage(dsn, keep_history=False, transform=None, reducer=None, **kw):
     """Create a RelStorage storage using the newt PostgresQL adapter.
 
     Keyword options can be used to provide either `ZODB.DB
@@ -118,6 +118,7 @@ def storage(dsn, keep_history=False, transform=None, **kw):
     options.
     """
     options = relstorage.options.Options(keep_history=keep_history, **kw)
+    options.reducer = reducer
     options.transform = transform
     return relstorage.storage.RelStorage(Adapter(dsn, options), options=options)
 

src/newt/db/component.py

@@ -5,16 +5,18 @@ class Adapter:
 
     def __init__(self, config):
         self.transform = config.transform
+        self.reducer = config.reducer
         self.config = config.adapter.config
 
     def create(self, options):
         from ._adapter import Adapter
-        transform = self.transform
-        if transform is not None:
-            mod, func = transform.rsplit('.', 1)
-            mod = __import__(mod, {}, {}, ['*'])
-            transform = getattr(mod, func)
-            options.transform = transform
+        for name in 'transform', 'reducer':
+            f = getattr(self, name)
+            if f is not None:
+                mod, func = f.rsplit('.', 1)
+                mod = __import__(mod, {}, {}, ['*'])
+                f = getattr(mod, func)
+                setattr(options, name, f)
 
         return Adapter(dsn=self.config.dsn, options=options)
 

src/newt/db/component.xml

@@ -27,6 +27,15 @@
       </description>
     </key>
 
+    <key name="reducer" datatype="dotted-name">
+      <description>
+        The dotted name of new.db.jsonpickle.JsonUnpickler reducer.
+
+        This is an advanced feature. See the docstring of that class
+        for details.
+      </description>
+    </key>
+
   </sectiontype>
 
   <sectiontype

src/newt/db/jsonpickle.py

@@ -180,6 +180,7 @@ def handle_set(args):
 
 def instance(global_, args):
     name = global_.name
+
     if name in special_classes:
         return special_classes[name](args)
 
@@ -203,20 +204,93 @@ class JsonUnpickler:
 
     Usage::
 
+      >>> import pickle
       >>> apickle = pickle.dumps([1,2])
       >>> unpickler = JsonUnpickler(apickle)
-      >>> json_string = unpickler.load()
+      >>> unpickler.load()
+      '[1, 2]'
       >>> unpickler.pos == len(apickle)
       True
+
+    **Very advanced** customization of special type handling:
+      You can supply an optional reducer to handle special built-in
+      objects or subclasses. For example, suppose we have a special
+      string class::
+
+        >>> class MySpecialString(str):
+        ...    pass
+
+      .. make pickleable:
+
+        >>> import newt.db.jsonpickle
+        >>> newt.db.jsonpickle.MySpecialString = MySpecialString
+
+      We can define a reducer that handles our special strings::
+
+        >>> def reducer(name, data):
+        ...     if name.endswith('MySpecialString'):
+        ...         return data if isinstance(data, str) else data[0]
+
+      A reducer will be called with a dotted class name and some data.
+      What the data is depends on the class.  It may be state, an
+      arguments tuple, or a tuple containing an argument tuple, and a
+      keyword argument dictionary.  The data may also contain
+      instances of objects defined by the ``newt.db.jsonpickle``
+      module, if so, you may just be able to use them.  If not and
+      they're instances of ``GET`` or ``PUT`` objects, then they wrap
+      data which you can get via ``v`` attributes. If that doesn't
+      work, you may just need to give up.
+
+      Reducers must return an object that is serializable by the
+      standard ``json`` module, or that has a ``json_reduce`` method
+      that returns an object that can be serialized by the ``json``
+      module.
+
+      Here's an example usage:
+
+        >>> special_string = MySpecialString('Hi')
+        >>> JsonUnpickler(pickle.dumps(special_string), reducer).load()
+        '"Hi"'
+
+      If our reducer doesn't handler a class, by returning None, this
+      data are handled as usual::
+
+        >>> from datetime import date
+        >>> JsonUnpickler(pickle.dumps(date(2017, 2, 27)), reducer).load()
+        '"2017-02-27"'
+
+      Use the :py:func:`dumps` function to experiment.
     """
 
     cyclic = False
 
-    def __init__(self, pickle):
+    def __init__(self, pickle, reducer=None):
         self.stack = []
         self.append = self.stack.append
         self.marks = []
         self.memo = {}
+
+        if reducer is None:
+            self.instance = instance
+        else:
+            def rinstance(global_, args):
+                name = global_.name
+                if name == 'copy_reg._reconstructor':
+                    # Gaaaa, special case for user-defined classes
+                    (cls, base, state) = args
+                    r = reducer(cls.name, state)
+                else:
+                    r = reducer(name, args)
+
+                if r is not None:
+                    return r
+
+                if name in special_classes:
+                    return special_classes[name](args)
+
+                return Instance(name, args)
+            self.instance = rinstance
+
         self.set_pickle(pickle)
 
     def set_pickle(self, pickle):
@@ -365,21 +439,22 @@ def STACK_GLOBAL(self, _):
 
     def REDUCE(self, _):
         f, args = self.pop(2)
-        self.append(instance(f,args))
+        self.append(self.instance(f,args))
 
     def BUILD(self, _):
         state = self.stack.pop()
         self.stack[-1].__setstate__(state)
 
     def INST(self, arg):
-        self.append(instance(Global(*arg.split()), tuple(self.pop_marked())))
+        self.append(self.instance(Global(*arg.split()),
+                                  tuple(self.pop_marked())))
 
     def OBJ(self, _):
         args = self.pop_marked()
-        self.append(instance(args[0], tuple(args[1:])))
+        self.append(self.instance(args[0], tuple(args[1:])))
 
     def NEWOBJ(self, _):
-        self.append(instance(*self.pop(2)))
+        self.append(self.instance(*self.pop(2)))
 
     def NEWOBJ_EX(self, _):
         cls, args, kw = self.pop(3)
@@ -388,7 +463,7 @@ def NEWOBJ_EX(self, _):
                 args = args, kw
             else:
                 args = kw
-        self.append(instance(cls, args))
+        self.append(self.instance(cls, args))
 
     def PROTO(self, _): pass
     def FRAME(self, _): pass
@@ -400,6 +475,31 @@ def PERSID(self, id):           # pragma: no cover
     def BINPERSID(self, _):
         self.stack[-1] = Persistent(self.stack[-1])
 
+def dumps(data, reducer=None, proto=3 if PY3 else 1):
+    """Dump an object to JSON using pickle and JsonUnpickler
+
+    This is useful for seeing how objects will be pickled, especially
+    when creating custon reducers.
+
+    Usage::
+
+        >>> dumps(42)
+        '42'
+
+    Note that the JSON produced is a little prettier than the default
+    JSON because keys are sorted and indentation is used::
+
+        >>> print(dumps(dict(a=1, b=2)))
+        {
+          "a": 1,
+          "b": 2
+        }
+
+    """
+    import pickle
+    return JsonUnpickler(pickle.dumps(data, proto), reducer).load(
+        sort_keys=True, indent=2).replace(' \n', '\n')
+
 unicode_surrogates = re.compile(r'\\ud[89a-f][0-9a-f]{2,2}', flags=re.I)
 NoneNoneNone = None, None, None
 
@@ -433,7 +533,7 @@ class Jsonifier:
     skip_class = re.compile('BTrees[.]|ZODB.blob').match
     skip = object() # marker
 
-    def __init__(self, skip_class=None, transform=None):
+    def __init__(self, skip_class=None, transform=None, reducer=None):
         """Create a callable for converting database data to Newt JSON
 
         Parameters:
@@ -467,6 +567,7 @@ def __init__(self, skip_class=None, transform=None):
         if skip_class is not None:
             self.skip_class = skip_class
         self.transform = transform
+        self.reducer = reducer
 
     def __call__(self, id, data):
         """Convert data from a ZODB data record to data used by newt.
@@ -489,7 +590,7 @@ def __call__(self, id, data):
         """
         if not data:
             return NoneNoneNone
-        unpickler = JsonUnpickler(data)
+        unpickler = JsonUnpickler(data, self.reducer)
         try:
             klass = json.loads(unpickler.load())
 

src/newt/db/tests/testdocs.py

@@ -35,7 +35,8 @@ def test_suite():
 
     p = lambda *names: os.path.join(doc, *names) + '.rst'
 
-    return manuel.testing.TestSuite(
+    return unittest.TestSuite((
+        manuel.testing.TestSuite(
             manuel.doctest.Manuel() + manuel.capture.Manuel(),
             p('fine-print'),
             p('getting-started'),
@@ -45,4 +46,6 @@ def test_suite():
             p('topics', 'zodburi'),
             p('topics', 'data-transformation'),
             setUp=setUp, tearDown=setupstack.tearDown,
-            )
+            ),
+        doctest.DocTestSuite('newt.db.jsonpickle'),
+        ))