Skip to content

[Python] Improve python builder performance #8766

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 1 commit into
base: master
Choose a base branch
from
+113 −81
Open

[Python] Improve python builder performance #8766

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
194 changes: 113 additions & 81 deletions python/flatbuffers/builder.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,6 @@
from . import packer
from .compat import memoryview_type
from .compat import NumpyRequiredForThisFeature, import_numpy
from .compat import range_func
from .number_types import (SOffsetTFlags, UOffsetTFlags, VOffsetTFlags)

np = import_numpy()
Expand Down Expand Up @@ -159,7 +158,7 @@ def __init__(self, initialSize=1024):
self.vtables = {}
self.nested = False
self.forceDefaults = False
self.sharedStrings = {}
self.sharedStrings = None
## @endcond
self.finished = False

Expand All @@ -172,7 +171,7 @@ def Clear(self):
self.vtables = {}
self.nested = False
self.forceDefaults = False
self.sharedStrings = {}
self.sharedStrings = None
self.vectorNumElems = None
## @endcond
self.finished = False
Expand All @@ -192,7 +191,7 @@ def Output(self):
if not self.finished:
raise BuilderNotFinishedError()

return self.Bytes[self.Head() :]
return self.Bytes[self.head :]

## @cond FLATBUFFERS_INTERNAL
def StartObject(self, numfields):
Expand All @@ -201,7 +200,7 @@ def StartObject(self, numfields):
self.assertNotNested()

# use 32-bit offsets so that arithmetic doesn't overflow.
self.current_vtable = [0 for _ in range_func(numfields)]
self.current_vtable = [0] * numfields
self.objectEnd = self.Offset()
self.nested = True

Expand Down Expand Up @@ -258,6 +257,7 @@ def WriteVtable(self):
i = len(self.current_vtable) - 1
trailing = 0
trim = True
vt_entries = []
while i >= 0:
off = 0
elem = self.current_vtable[i]
Expand All @@ -273,17 +273,24 @@ def WriteVtable(self):
off = objectOffset - elem
trim = False

self.PrependVOffsetT(off)
vt_entries.append(VOffsetTFlags.py_type(off))

# The two metadata fields are written last.

# First, store the object bytesize:
self.PrependVOffsetT(VOffsetTFlags.py_type(objectSize))
vt_entries.append(VOffsetTFlags.py_type(objectSize))

# Second, store the vtable bytesize:
vBytes = len(self.current_vtable) - trailing + VtableMetadataFields
vBytes *= N.VOffsetTFlags.bytewidth
self.PrependVOffsetT(VOffsetTFlags.py_type(vBytes))
vt_entries.append(VOffsetTFlags.py_type(vBytes))

field_entries = vt_entries[:-2]
field_entries.reverse()
ordered_entries = [vt_entries[-1], vt_entries[-2]]
ordered_entries.extend(field_entries)

self.WriteVtableEntries(ordered_entries)

# Next, write the offset to the new vtable in the
# already-allocated SOffsetT at the beginning of this object:
Expand All @@ -308,20 +315,37 @@ def WriteVtable(self):
encode.Write(
packer.soffset,
self.Bytes,
self.Head(),
self.head,
SOffsetTFlags.py_type(vt2Offset - objectOffset),
)

self.current_vtable = None
return objectOffset

def WriteVtableEntries(self, entries):
"""Write a contiguous block of VOffsetT values with a single prep call."""
count = len(entries)
if count == 0:
return
elem_size = N.VOffsetTFlags.bytewidth
total_bytes = elem_size * count
self.Prep(elem_size, total_bytes - elem_size)
head = self.head - total_bytes
self.head = UOffsetTFlags.py_type(head)
pack = packer.voffset.pack_into
buf = memoryview_type(self.Bytes)
offset = head
for value in entries:
pack(buf, offset, value)
offset += elem_size

def EndObject(self):
"""EndObject writes data necessary to finish object construction."""
self.assertNested()
self.nested = False
return self.WriteVtable()

def growByteBuffer(self):
def GrowByteBuffer(self):
"""Doubles the size of the byteslice, and copies the old data towards

the end of the new buffer (since we build the buffer backwards).
Expand Down Expand Up @@ -352,12 +376,15 @@ def Head(self):
## @cond FLATBUFFERS_INTERNAL
def Offset(self):
"""Offset relative to the end of the buffer."""
return UOffsetTFlags.py_type(len(self.Bytes) - self.Head())
return UOffsetTFlags.py_type(len(self.Bytes) - self.head)

def Pad(self, n):
"""Pad places zeros at the current offset."""
for i in range_func(n):
self.Place(0, N.Uint8Flags)
if n <= 0:
return
new_head = self.head - n
self.Bytes[new_head : self.head] = b"\x00" * n
self.head = UOffsetTFlags.py_type(new_head)

def Prep(self, size, additionalBytes):
"""Prep prepares to write an element of `size` after `additional_bytes`
Expand All @@ -374,15 +401,19 @@ def Prep(self, size, additionalBytes):

# Find the amount of alignment needed such that `size` is properly
# aligned after `additionalBytes`:
alignSize = (~(len(self.Bytes) - self.Head() + additionalBytes)) + 1
head = self.head
buf_len = len(self.Bytes)
alignSize = (~(buf_len - head + additionalBytes)) + 1
alignSize &= size - 1

# Reallocate the buffer if needed:
while self.Head() < alignSize + size + additionalBytes:
oldBufSize = len(self.Bytes)
self.growByteBuffer()
updated_head = self.head + len(self.Bytes) - oldBufSize
self.head = UOffsetTFlags.py_type(updated_head)
needed = alignSize + size + additionalBytes
while head < needed:
oldBufSize = buf_len
self.GrowByteBuffer()
buf_len = len(self.Bytes)
head += buf_len - oldBufSize
self.head = UOffsetTFlags.py_type(head)
self.Pad(alignSize)

def PrependSOffsetTRelative(self, off):
Expand Down Expand Up @@ -457,7 +488,9 @@ def CreateSharedString(self, s, encoding="utf-8", errors="strict"):
before calling CreateString.
"""

if s in self.sharedStrings:
if not self.sharedStrings:
self.sharedStrings = {}
elif s in self.sharedStrings:
return self.sharedStrings[s]

off = self.CreateString(s, encoding, errors)
Expand All @@ -480,16 +513,17 @@ def CreateString(self, s, encoding="utf-8", errors="strict"):
else:
raise TypeError("non-string passed to CreateString")

self.Prep(N.UOffsetTFlags.bytewidth, (len(x) + 1) * N.Uint8Flags.bytewidth)
payload_len = len(x)
self.Prep(
N.UOffsetTFlags.bytewidth, (payload_len + 1) * N.Uint8Flags.bytewidth
)
self.Place(0, N.Uint8Flags)

l = UOffsetTFlags.py_type(len(s))
## @cond FLATBUFFERS_INTERNAL
self.head = UOffsetTFlags.py_type(self.Head() - l)
## @endcond
self.Bytes[self.Head() : self.Head() + l] = x
new_head = self.head - payload_len
self.head = UOffsetTFlags.py_type(new_head)
self.Bytes[new_head : new_head + payload_len] = x

self.vectorNumElems = len(x)
self.vectorNumElems = payload_len
return self.EndVector()

def CreateByteVector(self, x):
Expand All @@ -503,15 +537,13 @@ def CreateByteVector(self, x):
if not isinstance(x, compat.binary_types):
raise TypeError("non-byte vector passed to CreateByteVector")

self.Prep(N.UOffsetTFlags.bytewidth, len(x) * N.Uint8Flags.bytewidth)
data_len = len(x)
self.Prep(N.UOffsetTFlags.bytewidth, data_len * N.Uint8Flags.bytewidth)
new_head = self.head - data_len
self.head = UOffsetTFlags.py_type(new_head)
self.Bytes[new_head : new_head + data_len] = x

l = UOffsetTFlags.py_type(len(x))
## @cond FLATBUFFERS_INTERNAL
self.head = UOffsetTFlags.py_type(self.Head() - l)
## @endcond
self.Bytes[self.Head() : self.Head() + l] = x

self.vectorNumElems = len(x)
self.vectorNumElems = data_len
return self.EndVector()

def CreateNumpyVector(self, x):
Expand All @@ -538,14 +570,14 @@ def CreateNumpyVector(self, x):
else:
x_lend = x.byteswap(inplace=False)

# Calculate total length
l = UOffsetTFlags.py_type(x_lend.itemsize * x_lend.size)
## @cond FLATBUFFERS_INTERNAL
self.head = UOffsetTFlags.py_type(self.Head() - l)
## @endcond

# tobytes ensures c_contiguous ordering
self.Bytes[self.Head() : self.Head() + l] = x_lend.tobytes(order="C")
payload = x_lend.tobytes(order="C")

# Calculate total length
payload_len = len(payload)
new_head = self.head - payload_len
self.head = UOffsetTFlags.py_type(new_head)
self.Bytes[new_head : new_head + payload_len] = payload

self.vectorNumElems = x.size
return self.EndVector()
Expand Down Expand Up @@ -615,11 +647,11 @@ def __Finish(self, rootTable, sizePrefix, file_identifier=None):

self.PrependUOffsetTRelative(rootTable)
if sizePrefix:
size = len(self.Bytes) - self.Head()
size = len(self.Bytes) - self.head
N.enforce_number(size, N.Int32Flags)
self.PrependInt32(size)
self.finished = True
return self.Head()
return self.head

def Finish(self, rootTable, file_identifier=None):
"""Finish finalizes a buffer, pointing to the given `rootTable`."""
Expand All @@ -634,8 +666,31 @@ def FinishSizePrefixed(self, rootTable, file_identifier=None):

## @cond FLATBUFFERS_INTERNAL
def Prepend(self, flags, off):
self.Prep(flags.bytewidth, 0)
self.Place(off, flags)
size = flags.bytewidth
if size > self.minalign:
self.minalign = size

head = self.head
buf_len = len(self.Bytes)
alignSize = (~(buf_len - head)) + 1
alignSize &= size - 1

needed = alignSize + size
while head < needed:
oldBufSize = buf_len
self.GrowByteBuffer()
buf_len = len(self.Bytes)
head += buf_len - oldBufSize

if alignSize:
new_head = head - alignSize
self.Bytes[new_head:head] = b"\x00" * alignSize
head = new_head

N.enforce_number(off, flags)
head -= size
self.head = UOffsetTFlags.py_type(head)
encode.Write(flags.packer_type, self.Bytes, head, off)

def PrependSlot(self, flags, o, x, d):
if x is not None:
Expand Down Expand Up @@ -803,70 +858,47 @@ def ForceDefaults(self, forceDefaults):
##############################################################

## @cond FLATBUFFERS_INTERNAL
def PrependVOffsetT(self, x):
self.Prepend(N.VOffsetTFlags, x)

def Place(self, x, flags):
"""Place prepends a value specified by `flags` to the Builder,

without checking for available space.
"""

N.enforce_number(x, flags)
self.head = self.head - flags.bytewidth
encode.Write(flags.packer_type, self.Bytes, self.Head(), x)
new_head = self.head - flags.bytewidth
self.head = UOffsetTFlags.py_type(new_head)
encode.Write(flags.packer_type, self.Bytes, new_head, x)

def PlaceVOffsetT(self, x):
"""PlaceVOffsetT prepends a VOffsetT to the Builder, without checking

for space.
"""
N.enforce_number(x, N.VOffsetTFlags)
self.head = self.head - N.VOffsetTFlags.bytewidth
encode.Write(packer.voffset, self.Bytes, self.Head(), x)
new_head = self.head - N.VOffsetTFlags.bytewidth
self.head = UOffsetTFlags.py_type(new_head)
encode.Write(packer.voffset, self.Bytes, new_head, x)

def PlaceSOffsetT(self, x):
"""PlaceSOffsetT prepends a SOffsetT to the Builder, without checking

for space.
"""
N.enforce_number(x, N.SOffsetTFlags)
self.head = self.head - N.SOffsetTFlags.bytewidth
encode.Write(packer.soffset, self.Bytes, self.Head(), x)
new_head = self.head - N.SOffsetTFlags.bytewidth
self.head = UOffsetTFlags.py_type(new_head)
encode.Write(packer.soffset, self.Bytes, new_head, x)

def PlaceUOffsetT(self, x):
"""PlaceUOffsetT prepends a UOffsetT to the Builder, without checking

for space.
"""
N.enforce_number(x, N.UOffsetTFlags)
self.head = self.head - N.UOffsetTFlags.bytewidth
encode.Write(packer.uoffset, self.Bytes, self.Head(), x)
new_head = self.head - N.UOffsetTFlags.bytewidth
self.head = UOffsetTFlags.py_type(new_head)
encode.Write(packer.uoffset, self.Bytes, new_head, x)

## @endcond


## @cond FLATBUFFERS_INTERNAL
def vtableEqual(a, objectStart, b):
"""vtableEqual compares an unwritten vtable to a written vtable."""

N.enforce_number(objectStart, N.UOffsetTFlags)

if len(a) * N.VOffsetTFlags.bytewidth != len(b):
return False

for i, elem in enumerate(a):
x = encode.Get(packer.voffset, b, i * N.VOffsetTFlags.bytewidth)

# Skip vtable entries that indicate a default value.
if x == 0 and elem == 0:
pass
else:
y = objectStart - elem
if x != y:
return False
return True


## @endcond
## @}