SL-19314: Recast llsd serialization to write to a stream.

llsd/__init__.py

@@ -11,9 +11,13 @@
                        BINARY_HEADER, NOTATION_HEADER, XML_HEADER,
                        LLSDBaseParser, LLSDParseError, LLSDSerializationError,
                        LongType, UnicodeType, binary, undef, uri)
-from llsd.serde_binary import LLSDBinaryParser, format_binary, parse_binary, parse_binary_nohdr
-from llsd.serde_notation import LLSDNotationFormatter, LLSDNotationParser, format_notation, parse_notation, parse_notation_nohdr
-from llsd.serde_xml import LLSDXMLFormatter, LLSDXMLPrettyFormatter, format_pretty_xml, format_xml, parse_xml, parse_xml_nohdr
+from llsd.serde_binary import (LLSDBinaryParser, format_binary, parse_binary, parse_binary_nohdr,
+                               write_binary)
+from llsd.serde_notation import (LLSDNotationFormatter, write_notation, format_notation,
+                                 LLSDNotationParser, parse_notation, parse_notation_nohdr)
+from llsd.serde_xml import (LLSDXMLFormatter, LLSDXMLPrettyFormatter,
+                            write_pretty_xml, write_xml, format_pretty_xml, format_xml,
+                            parse_xml, parse_xml_nohdr)
 
 
 def parse(something, mime_type = None):

llsd/base.py

@@ -42,6 +42,8 @@ def __init__(self, thing=None):
 undef = _LLSD(None)
 
 
+# 'binary' only exists so that a Python 2 caller can distinguish binary data
+# from str data - since in Python 2, (bytes is str).
 if PY2:
     class binary(str):
         "Simple wrapper for llsd.binary data."
@@ -174,7 +176,7 @@ def _format_datestr(v):
     xml and notation serializations.
     """
     if not isinstance(v, datetime.date) and not isinstance(v, datetime.datetime):
-        raise LLSDParseError("invalid date string %s passed to date formatter" % v)
+        raise LLSDSerializationError("invalid date string %s passed to date formatter" % v)
 
     if not isinstance(v, datetime.datetime):
         v = datetime.datetime.combine(v, datetime.time(0))
@@ -322,8 +324,11 @@ class LLSDBaseFormatter(object):
     role of this base class is to provide self.type_map based on the methods
     defined in its subclass.
     """
+    __slots__ = ['stream', 'type_map']
+
     def __init__(self):
         "Construct a new formatter dispatch table."
+        self.stream = None
         self.type_map = {
             type(None):          self.UNDEF,
             undef:               self.UNDEF,
@@ -347,6 +352,33 @@ def __init__(self):
         }
 
 
+    def format(self, something):
+        """
+        Pure Python implementation of the formatter.
+        Format a python object according to subclass formatting.
+
+        :param something: A python object (typically a dict) to be serialized.
+        :returns: A serialized bytes object.
+        """
+        stream = io.BytesIO()
+        self.write(stream, something)
+        return stream.getvalue()
+
+    def write(self, stream, something):
+        """
+        Serialize a python object to the passed binary 'stream' according to
+        subclass formatting.
+
+        :param stream: A binary file-like object to which to serialize 'something'.
+        :param something: A python object (typically a dict) to be serialized.
+        """
+        self.stream = stream
+        try:
+            return self._write(something)
+        finally:
+            self.stream = None
+
+
 _X_ORD = ord(b'x')
 _BACKSLASH_ORD = ord(b'\\')
 _DECODE_BUFF_ALLOC_SIZE = 1024

llsd/serde_binary.py

@@ -1,5 +1,6 @@
 import calendar
 import datetime
+import io
 import struct
 import uuid
 
@@ -152,65 +153,65 @@ def format_binary(something):
    :param something: a python object (typically a dict) to be serialized.
    :returns: Returns a LLSD binary formatted string.
     """
-    return BINARY_HEADER + b'\n' + _format_binary_recurse(something)
+    stream = io.BytesIO()
+    write_binary(stream, something)
+    return stream.getvalue()
 
 
-def _format_binary_recurse(something):
+def write_binary(stream, something):
+    stream.write(b'<?llsd/binary?>\n')
+    _write_binary_recurse(stream, something)
+
+
+def _write_binary_recurse(stream, something):
     "Binary formatter workhorse."
     def _format_list(something):
-        array_builder = []
-        array_builder.append(b'[' + struct.pack('!i', len(something)))
+        stream.writelines([b'[', struct.pack('!i', len(something))])
         for item in something:
-            array_builder.append(_format_binary_recurse(item))
-        array_builder.append(b']')
-        return b''.join(array_builder)
+            _write_binary_recurse(stream, item)
+        stream.write(b']')
 
     if something is None:
-        return b'!'
+        stream.write(b'!')
     elif isinstance(something, _LLSD):
-        return _format_binary_recurse(something.thing)
+        _write_binary_recurse(stream, something.thing)
     elif isinstance(something, bool):
-        if something:
-            return b'1'
-        else:
-            return b'0'
+        stream.write(b'1' if something else b'0')
     elif is_integer(something):
         try:
-            return b'i' + struct.pack('!i', something)
+            stream.writelines([b'i', struct.pack('!i', something)])
         except (OverflowError, struct.error) as exc:
             raise LLSDSerializationError(str(exc), something)
     elif isinstance(something, float):
         try:
-            return b'r' + struct.pack('!d', something)
+            stream.writelines([b'r', struct.pack('!d', something)])
         except SystemError as exc:
             raise LLSDSerializationError(str(exc), something)
     elif isinstance(something, uuid.UUID):
-        return b'u' + something.bytes
+        stream.writelines([b'u', something.bytes])
     elif isinstance(something, binary):
-        return b'b' + struct.pack('!i', len(something)) + something
+        stream.writelines([b'b', struct.pack('!i', len(something)), something])
     elif is_string(something):
         something = _str_to_bytes(something)
-        return b's' + struct.pack('!i', len(something)) + something
+        stream.writelines([b's', struct.pack('!i', len(something)), something])
     elif isinstance(something, uri):
-        return b'l' + struct.pack('!i', len(something)) + something
+        stream.writelines([b'l', struct.pack('!i', len(something)), something])
     elif isinstance(something, datetime.datetime):
         seconds_since_epoch = calendar.timegm(something.utctimetuple()) \
                               + something.microsecond // 1e6
-        return b'd' + struct.pack('<d', seconds_since_epoch)
+        stream.writelines([b'd', struct.pack('<d', seconds_since_epoch)])
     elif isinstance(something, datetime.date):
         seconds_since_epoch = calendar.timegm(something.timetuple())
-        return b'd' + struct.pack('<d', seconds_since_epoch)
+        stream.writelines([b'd', struct.pack('<d', seconds_since_epoch)])
     elif isinstance(something, (list, tuple)):
-        return _format_list(something)
+        _format_list(something)
     elif isinstance(something, dict):
-        map_builder = []
-        map_builder.append(b'{' + struct.pack('!i', len(something)))
+        stream.writelines([b'{', struct.pack('!i', len(something))])
         for key, value in something.items():
             key = _str_to_bytes(key)
-            map_builder.append(b'k' + struct.pack('!i', len(key)) + key)
-            map_builder.append(_format_binary_recurse(value))
-        map_builder.append(b'}')
-        return b''.join(map_builder)
+            stream.writelines([b'k', struct.pack('!i', len(key)), key])
+            _write_binary_recurse(stream, value)
+        stream.write(b'}')
     else:
         try:
             return _format_list(list(something))

llsd/serde_notation.py

@@ -346,21 +346,16 @@ class LLSDNotationFormatter(LLSDBaseFormatter):
 
     See http://wiki.secondlife.com/wiki/LLSD#Notation_Serialization
     """
-    __slots__ = []
-
     def LLSD(self, v):
         return self._generate(v.thing)
     def UNDEF(self, v):
-        return b'!'
+        self.stream.write(b'!')
     def BOOLEAN(self, v):
-        if v:
-            return b'true'
-        else:
-            return b'false'
+        self.stream.write(b'true' if v else b'false')
     def INTEGER(self, v):
-        return B("i%d") % v
+        self.stream.write(B("i%d") % v)
     def REAL(self, v):
-        return B("r%r") % v
+        self.stream.write(B("r%r") % v)
     def UUID(self, v):
         # latin-1 is the byte-to-byte encoding, mapping \x00-\xFF ->
         # \u0000-\u00FF. It's also the fastest encoding, I believe, from
@@ -370,24 +365,42 @@ def UUID(self, v):
         # error behavior in case someone passes an invalid hex string, with
         # things other than 0-9a-fA-F, so that they will fail in the UUID
         # decode, rather than with a UnicodeError.
-        return B("u%s") % str(v).encode('latin-1')
+        self.stream.writelines([b"u", str(v).encode('latin-1')])
     def BINARY(self, v):
-        return b'b64"' + base64.b64encode(v).strip() + b'"'
+        self.stream.writelines([b'b64"', base64.b64encode(v).strip(), b'"'])
 
     def STRING(self, v):
-        return B("'%s'") % _str_to_bytes(v).replace(b"\\", b"\\\\").replace(b"'", b"\\'")
+        self.stream.writelines([b"'", self._esc(v), b"'"])
     def URI(self, v):
-        return B('l"%s"') % _str_to_bytes(v).replace(b"\\", b"\\\\").replace(b'"', b'\\"')
+        self.stream.writelines([b'l"', self._esc(v, b'"'), b'"'])
     def DATE(self, v):
-        return B('d"%s"') % _format_datestr(v)
+        self.stream.writelines([b'd"', _format_datestr(v), b'"'])
     def ARRAY(self, v):
-        return B("[%s]") % b','.join([self._generate(item) for item in v])
+        self.stream.write(b'[')
+        delim = b''
+        for item in v:
+            self.stream.write(delim)
+            self._generate(item)
+            delim = b','
+        self.stream.write(b']')
     def MAP(self, v):
-        return B("{%s}") % b','.join([B("'%s':%s") % (_str_to_bytes(UnicodeType(key)).replace(b"\\", b"\\\\").replace(b"'", b"\\'"), self._generate(value))
-             for key, value in v.items()])
+        self.stream.write(b'{')
+        delim = b''
+        for key, value in v.items():
+            self.stream.writelines([delim, b"'", self._esc(UnicodeType(key)), b"':"])
+            self._generate(value)
+            delim = b','
+        self.stream.write(b'}')
+
+    def _esc(self, data, quote=b"'"):
+        return _str_to_bytes(data).replace(b"\\", b"\\\\").replace(quote, b'\\'+quote)
 
     def _generate(self, something):
-        "Generate notation from a single python object."
+        """
+        Serialize a python object to self.stream as application/llsd+notation
+
+        :param something: a python object (typically a dict) to be serialized.
+        """
         t = type(something)
         handler = self.type_map.get(t)
         if handler:
@@ -401,14 +414,8 @@ def _generate(self, something):
                 raise LLSDSerializationError(
                     "Cannot serialize unknown type: %s (%s)" % (t, something))
 
-    def format(self, something):
-        """
-        Format a python object as application/llsd+notation
-
-        :param something: a python object (typically a dict) to be serialized.
-        :returns: Returns a LLSD notation formatted string.
-        """
-        return self._generate(something)
+    # _write() method is an alias for _generate()
+    _write = _generate
 
 
 def format_notation(something):
@@ -423,6 +430,32 @@ def format_notation(something):
     return LLSDNotationFormatter().format(something)
 
 
+def write_notation(stream, something):
+    """
+    Serialize to passed binary 'stream' a python object 'something' as
+    application/llsd+notation.
+
+    :param stream: a binary stream open for writing.
+    :param something: a python object (typically a dict) to be serialized.
+
+    See http://wiki.secondlife.com/wiki/LLSD#Notation_Serialization
+    """
+    return LLSDNotationFormatter().write(stream, something)
+
+
+def write_notation(stream, something):
+    """
+    Serialize to passed binary 'stream' a python object 'something' as
+    application/llsd+notation.
+
+    :param stream: a binary stream open for writing.
+    :param something: a python object (typically a dict) to be serialized.
+
+    See http://wiki.secondlife.com/wiki/LLSD#Notation_Serialization
+    """
+    return LLSDNotationFormatter().write(stream, something)
+
+
 def parse_notation(something):
     """
     This is the basic public interface for parsing llsd+notation.