Increase python performance

- Builder startup work is cheaper: StartObject now seeds vtable state
  with [0] * numfields, lazy create sharedStrings dict to speed up
  objects with no strings

- Offset/Pad/Prep all work off cached head/buffer lengths and zero-fill
  via slices

- Prepend now handles alignment + byte writes in one pass

- Vtable write is batched: WriteVtable gathers all field offsets plus
  metadata and streams them

Author: satur9nine

python/flatbuffers/builder.py

@@ -20,7 +20,6 @@
 from . import packer
 from .compat import memoryview_type
 from .compat import NumpyRequiredForThisFeature, import_numpy
-from .compat import range_func
 from .number_types import (SOffsetTFlags, UOffsetTFlags, VOffsetTFlags)
 
 np = import_numpy()
@@ -159,7 +158,7 @@ def __init__(self, initialSize=1024):
     self.vtables = {}
     self.nested = False
     self.forceDefaults = False
-    self.sharedStrings = {}
+    self.sharedStrings = None
     ## @endcond
     self.finished = False
 
@@ -172,7 +171,7 @@ def Clear(self) -> None:
     self.vtables = {}
     self.nested = False
     self.forceDefaults = False
-    self.sharedStrings = {}
+    self.sharedStrings = None
     self.vectorNumElems = None
     ## @endcond
     self.finished = False
@@ -192,7 +191,7 @@ def Output(self):
     if not self.finished:
       raise BuilderNotFinishedError()
 
-    return self.Bytes[self.Head() :]
+    return self.Bytes[self.head :]
 
   ## @cond FLATBUFFERS_INTERNAL
   def StartObject(self, numfields):
@@ -201,7 +200,7 @@ def StartObject(self, numfields):
     self.assertNotNested()
 
     # use 32-bit offsets so that arithmetic doesn't overflow.
-    self.current_vtable = [0 for _ in range_func(numfields)]
+    self.current_vtable = [0] * numfields
     self.objectEnd = self.Offset()
     self.nested = True
 
@@ -258,6 +257,7 @@ def WriteVtable(self):
       i = len(self.current_vtable) - 1
       trailing = 0
       trim = True
+      vt_entries = []
       while i >= 0:
         off = 0
         elem = self.current_vtable[i]
@@ -273,17 +273,24 @@ def WriteVtable(self):
           off = objectOffset - elem
           trim = False
 
-        self.PrependVOffsetT(off)
+        vt_entries.append(VOffsetTFlags.py_type(off))
 
       # The two metadata fields are written last.
 
       # First, store the object bytesize:
-      self.PrependVOffsetT(VOffsetTFlags.py_type(objectSize))
+      vt_entries.append(VOffsetTFlags.py_type(objectSize))
 
       # Second, store the vtable bytesize:
       vBytes = len(self.current_vtable) - trailing + VtableMetadataFields
       vBytes *= N.VOffsetTFlags.bytewidth
-      self.PrependVOffsetT(VOffsetTFlags.py_type(vBytes))
+      vt_entries.append(VOffsetTFlags.py_type(vBytes))
+
+      field_entries = vt_entries[:-2]
+      field_entries.reverse()
+      ordered_entries = [vt_entries[-1], vt_entries[-2]]
+      ordered_entries.extend(field_entries)
+
+      self.WriteVtableEntries(ordered_entries)
 
       # Next, write the offset to the new vtable in the
       # already-allocated SOffsetT at the beginning of this object:
@@ -308,20 +315,37 @@ def WriteVtable(self):
       encode.Write(
           packer.soffset,
           self.Bytes,
-          self.Head(),
+          self.head,
           SOffsetTFlags.py_type(vt2Offset - objectOffset),
       )
 
     self.current_vtable = None
     return objectOffset
 
+  def WriteVtableEntries(self, entries):
+    """Write a contiguous block of VOffsetT values with a single prep call."""
+    count = len(entries)
+    if count == 0:
+      return
+    elem_size = N.VOffsetTFlags.bytewidth
+    total_bytes = elem_size * count
+    self.Prep(elem_size, total_bytes - elem_size)
+    head = self.head - total_bytes
+    self.head = UOffsetTFlags.py_type(head)
+    pack = packer.voffset.pack_into
+    buf = memoryview_type(self.Bytes)
+    offset = head
+    for value in entries:
+      pack(buf, offset, value)
+      offset += elem_size
+
   def EndObject(self):
     """EndObject writes data necessary to finish object construction."""
     self.assertNested()
     self.nested = False
     return self.WriteVtable()
 
-  def growByteBuffer(self):
+  def GrowByteBuffer(self):
     """Doubles the size of the byteslice, and copies the old data towards
 
     the end of the new buffer (since we build the buffer backwards).
@@ -352,12 +376,15 @@ def Head(self):
   ## @cond FLATBUFFERS_INTERNAL
   def Offset(self):
     """Offset relative to the end of the buffer."""
-    return UOffsetTFlags.py_type(len(self.Bytes) - self.Head())
+    return UOffsetTFlags.py_type(len(self.Bytes) - self.head)
 
   def Pad(self, n):
     """Pad places zeros at the current offset."""
-    for i in range_func(n):
-      self.Place(0, N.Uint8Flags)
+    if n <= 0:
+      return
+    new_head = self.head - n
+    self.Bytes[new_head : self.head] = b"\x00" * n
+    self.head = UOffsetTFlags.py_type(new_head)
 
   def Prep(self, size, additionalBytes):
     """Prep prepares to write an element of `size` after `additional_bytes`
@@ -374,15 +401,19 @@ def Prep(self, size, additionalBytes):
 
     # Find the amount of alignment needed such that `size` is properly
     # aligned after `additionalBytes`:
-    alignSize = (~(len(self.Bytes) - self.Head() + additionalBytes)) + 1
+    head = self.head
+    buf_len = len(self.Bytes)
+    alignSize = (~(buf_len - head + additionalBytes)) + 1
     alignSize &= size - 1
 
     # Reallocate the buffer if needed:
-    while self.Head() < alignSize + size + additionalBytes:
-      oldBufSize = len(self.Bytes)
-      self.growByteBuffer()
-      updated_head = self.head + len(self.Bytes) - oldBufSize
-      self.head = UOffsetTFlags.py_type(updated_head)
+    needed = alignSize + size + additionalBytes
+    while head < needed:
+      oldBufSize = buf_len
+      self.GrowByteBuffer()
+      buf_len = len(self.Bytes)
+      head += buf_len - oldBufSize
+    self.head = UOffsetTFlags.py_type(head)
     self.Pad(alignSize)
 
   def PrependSOffsetTRelative(self, off):
@@ -457,7 +488,9 @@ def CreateSharedString(self, s, encoding="utf-8", errors="strict"):
     before calling CreateString.
     """
 
-    if s in self.sharedStrings:
+    if not self.sharedStrings:
+      self.sharedStrings = {}
+    elif s in self.sharedStrings:
       return self.sharedStrings[s]
 
     off = self.CreateString(s, encoding, errors)
@@ -480,16 +513,17 @@ def CreateString(self, s, encoding="utf-8", errors="strict"):
     else:
       raise TypeError("non-string passed to CreateString")
 
-    self.Prep(N.UOffsetTFlags.bytewidth, (len(x) + 1) * N.Uint8Flags.bytewidth)
+    payload_len = len(x)
+    self.Prep(
+      N.UOffsetTFlags.bytewidth, (payload_len + 1) * N.Uint8Flags.bytewidth
+    )
     self.Place(0, N.Uint8Flags)
 
-    l = UOffsetTFlags.py_type(len(s))
-    ## @cond FLATBUFFERS_INTERNAL
-    self.head = UOffsetTFlags.py_type(self.Head() - l)
-    ## @endcond
-    self.Bytes[self.Head() : self.Head() + l] = x
+    new_head = self.head - payload_len
+    self.head = UOffsetTFlags.py_type(new_head)
+    self.Bytes[new_head : new_head + payload_len] = x
 
-    self.vectorNumElems = len(x)
+    self.vectorNumElems = payload_len
     return self.EndVector()
 
   def CreateByteVector(self, x):
@@ -503,15 +537,13 @@ def CreateByteVector(self, x):
     if not isinstance(x, compat.binary_types):
       raise TypeError("non-byte vector passed to CreateByteVector")
 
-    self.Prep(N.UOffsetTFlags.bytewidth, len(x) * N.Uint8Flags.bytewidth)
+    data_len = len(x)
+    self.Prep(N.UOffsetTFlags.bytewidth, data_len * N.Uint8Flags.bytewidth)
+    new_head = self.head - data_len
+    self.head = UOffsetTFlags.py_type(new_head)
+    self.Bytes[new_head : new_head + data_len] = x
 
-    l = UOffsetTFlags.py_type(len(x))
-    ## @cond FLATBUFFERS_INTERNAL
-    self.head = UOffsetTFlags.py_type(self.Head() - l)
-    ## @endcond
-    self.Bytes[self.Head() : self.Head() + l] = x
-
-    self.vectorNumElems = len(x)
+    self.vectorNumElems = data_len
     return self.EndVector()
 
   def CreateNumpyVector(self, x):
@@ -538,14 +570,14 @@ def CreateNumpyVector(self, x):
     else:
       x_lend = x.byteswap(inplace=False)
 
-    # Calculate total length
-    l = UOffsetTFlags.py_type(x_lend.itemsize * x_lend.size)
-    ## @cond FLATBUFFERS_INTERNAL
-    self.head = UOffsetTFlags.py_type(self.Head() - l)
-    ## @endcond
-
     # tobytes ensures c_contiguous ordering
-    self.Bytes[self.Head() : self.Head() + l] = x_lend.tobytes(order="C")
+    payload = x_lend.tobytes(order="C")
+
+    # Calculate total length
+    payload_len = len(payload)
+    new_head = self.head - payload_len
+    self.head = UOffsetTFlags.py_type(new_head)
+    self.Bytes[new_head : new_head + payload_len] = payload
 
     self.vectorNumElems = x.size
     return self.EndVector()
@@ -615,11 +647,11 @@ def __Finish(self, rootTable, sizePrefix, file_identifier=None):
 
     self.PrependUOffsetTRelative(rootTable)
     if sizePrefix:
-      size = len(self.Bytes) - self.Head()
+      size = len(self.Bytes) - self.head
       N.enforce_number(size, N.Int32Flags)
       self.PrependInt32(size)
     self.finished = True
-    return self.Head()
+    return self.head
 
   def Finish(self, rootTable, file_identifier=None):
     """Finish finalizes a buffer, pointing to the given `rootTable`."""
@@ -634,8 +666,31 @@ def FinishSizePrefixed(self, rootTable, file_identifier=None):
 
   ## @cond FLATBUFFERS_INTERNAL
   def Prepend(self, flags, off):
-    self.Prep(flags.bytewidth, 0)
-    self.Place(off, flags)
+    size = flags.bytewidth
+    if size > self.minalign:
+      self.minalign = size
+
+    head = self.head
+    buf_len = len(self.Bytes)
+    alignSize = (~(buf_len - head)) + 1
+    alignSize &= size - 1
+
+    needed = alignSize + size
+    while head < needed:
+      oldBufSize = buf_len
+      self.GrowByteBuffer()
+      buf_len = len(self.Bytes)
+      head += buf_len - oldBufSize
+
+    if alignSize:
+      new_head = head - alignSize
+      self.Bytes[new_head:head] = b"\x00" * alignSize
+      head = new_head
+
+    N.enforce_number(off, flags)
+    head -= size
+    self.head = UOffsetTFlags.py_type(head)
+    encode.Write(flags.packer_type, self.Bytes, head, off)
 
   def PrependSlot(self, flags, o, x, d):
     if x is not None:
@@ -803,70 +858,47 @@ def ForceDefaults(self, forceDefaults):
   ##############################################################
 
   ## @cond FLATBUFFERS_INTERNAL
-  def PrependVOffsetT(self, x):
-    self.Prepend(N.VOffsetTFlags, x)
-
   def Place(self, x, flags):
     """Place prepends a value specified by `flags` to the Builder,
 
     without checking for available space.
     """
 
     N.enforce_number(x, flags)
-    self.head = self.head - flags.bytewidth
-    encode.Write(flags.packer_type, self.Bytes, self.Head(), x)
+    new_head = self.head - flags.bytewidth
+    self.head = UOffsetTFlags.py_type(new_head)
+    encode.Write(flags.packer_type, self.Bytes, new_head, x)
 
   def PlaceVOffsetT(self, x):
     """PlaceVOffsetT prepends a VOffsetT to the Builder, without checking
 
     for space.
     """
     N.enforce_number(x, N.VOffsetTFlags)
-    self.head = self.head - N.VOffsetTFlags.bytewidth
-    encode.Write(packer.voffset, self.Bytes, self.Head(), x)
+    new_head = self.head - N.VOffsetTFlags.bytewidth
+    self.head = UOffsetTFlags.py_type(new_head)
+    encode.Write(packer.voffset, self.Bytes, new_head, x)
 
   def PlaceSOffsetT(self, x):
     """PlaceSOffsetT prepends a SOffsetT to the Builder, without checking
 
     for space.
     """
     N.enforce_number(x, N.SOffsetTFlags)
-    self.head = self.head - N.SOffsetTFlags.bytewidth
-    encode.Write(packer.soffset, self.Bytes, self.Head(), x)
+    new_head = self.head - N.SOffsetTFlags.bytewidth
+    self.head = UOffsetTFlags.py_type(new_head)
+    encode.Write(packer.soffset, self.Bytes, new_head, x)
 
   def PlaceUOffsetT(self, x):
     """PlaceUOffsetT prepends a UOffsetT to the Builder, without checking
 
     for space.
     """
     N.enforce_number(x, N.UOffsetTFlags)
-    self.head = self.head - N.UOffsetTFlags.bytewidth
-    encode.Write(packer.uoffset, self.Bytes, self.Head(), x)
+    new_head = self.head - N.UOffsetTFlags.bytewidth
+    self.head = UOffsetTFlags.py_type(new_head)
+    encode.Write(packer.uoffset, self.Bytes, new_head, x)
 
   ## @endcond
 
-
-## @cond FLATBUFFERS_INTERNAL
-def vtableEqual(a, objectStart, b):
-  """vtableEqual compares an unwritten vtable to a written vtable."""
-
-  N.enforce_number(objectStart, N.UOffsetTFlags)
-
-  if len(a) * N.VOffsetTFlags.bytewidth != len(b):
-    return False
-
-  for i, elem in enumerate(a):
-    x = encode.Get(packer.voffset, b, i * N.VOffsetTFlags.bytewidth)
-
-    # Skip vtable entries that indicate a default value.
-    if x == 0 and elem == 0:
-      pass
-    else:
-      y = objectStart - elem
-      if x != y:
-        return False
-  return True
-
-
-## @endcond
 ## @}