Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

New pyln modules. #3733

Merged
merged 18 commits into from
Jun 12, 2020
Merged

New pyln modules. #3733

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
18 commits
Select commit Hold shift + click to select a range
ca71845
pyln: add Makefile
rustyrussell May 18, 2020
31fa55e
pyln: add pyln.proto.message.
rustyrussell May 28, 2020
16297af
patch message-export-types.patch
rustyrussell Jun 3, 2020
1bff330
pyln.proto.message: use BufferedIOBase instead of bytes for binary ops.
rustyrussell Jun 4, 2020
46151ed
pyln.proto.message: expose fundamental MessageTypes as variables.
rustyrussell Jun 4, 2020
e274226
pyln: add (undocumented) u8 fundamental type.
rustyrussell Jun 4, 2020
59e2064
message: support option fields.
rustyrussell Jun 4, 2020
784d138
pyln.proto.message: separate fundamental types from other subtypes.
rustyrussell Jun 4, 2020
a4eb933
pyln: new module pyln.proto.message.bolts
rustyrussell Jun 4, 2020
42fc48f
new modules: pyln.proto.message.{bolt1,bolt2,bolt4,bolt7}
rustyrussell Jun 4, 2020
3ed6831
pyln.proto.message: support adding two namespaces.
rustyrussell Jun 4, 2020
360bcaf
pyln.proto.message: python-fluency feedback from @darosior
rustyrussell Jun 4, 2020
4a336db
pyln.proto.message: export more.
rustyrussell Jun 4, 2020
efd38a4
pyln.proto.message: allow fields with options to be missing.
rustyrussell Jun 4, 2020
bef68d3
pyln.proto.message.*: Add Makefile to do mypy checks.
rustyrussell Jun 4, 2020
851122d
pyln.proto.message.*: add type annotations.
rustyrussell Jun 12, 2020
e0d3174
pyln.proto.message: expose array types, add set_field for Message class.
rustyrussell Jun 12, 2020
5dbec8f
pyln.proto.message: fix handling of missing optional fields.
rustyrussell Jun 12, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
62 changes: 30 additions & 32 deletions contrib/pyln-proto/pyln/proto/message/array_types.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -42,28 +42,26 @@ def val_to_str(self, v, otherfields):

return '[' + s + ']'

def val_to_bin(self, v, otherfields):
b = bytes()
def write(self, io_out, v, otherfields):
for i in v:
b += self.elemtype.val_to_bin(i, otherfields)
return b
self.elemtype.write(io_out, i, otherfields)

def arr_from_bin(self, bytestream, otherfields, arraysize):
"""arraysize None means take rest of bytestream exactly"""
totsize = 0
def read_arr(self, io_in, otherfields, arraysize):
"""arraysize None means take rest of io entirely and exactly"""
vals = []
i = 0
while True:
if arraysize is None and totsize == len(bytestream):
return vals, totsize
elif i == arraysize:
return vals, totsize
val, size = self.elemtype.val_from_bin(bytestream[totsize:],
otherfields)
totsize += size
i += 1
while arraysize is None or len(vals) < arraysize:
# Throws an exception on partial read, so None means completely empty.
val = self.elemtype.read(io_in, otherfields)
if val is None:
if arraysize is not None:
raise ValueError('{}: not enough remaining to read'
.format(self))
break

vals.append(val)

return vals


class SizedArrayType(ArrayType):
"""A fixed-size array"""
Expand All @@ -82,13 +80,13 @@ def val_from_str(self, s):
raise ValueError("Length of {} != {}", s, self.arraysize)
return a, b

def val_to_bin(self, v, otherfields):
def write(self, io_out, v, otherfields):
if len(v) != self.arraysize:
raise ValueError("Length of {} != {}", v, self.arraysize)
return super().val_to_bin(v, otherfields)
return super().write(io_out, v, otherfields)

def val_from_bin(self, bytestream, otherfields):
return super().arr_from_bin(bytestream, otherfields, self.arraysize)
def read(self, io_in, otherfields):
return super().read_arr(io_in, otherfields, self.arraysize)


class EllipsisArrayType(ArrayType):
Expand All @@ -97,9 +95,9 @@ class EllipsisArrayType(ArrayType):
def __init__(self, tlv, name, elemtype):
super().__init__(tlv, name, elemtype)

def val_from_bin(self, bytestream, otherfields):
def read(self, io_in, otherfields):
"""Takes rest of bytestream"""
return super().arr_from_bin(bytestream, otherfields, None)
return super().read_arr(io_in, otherfields, None)

def only_at_tlv_end(self):
"""These only make sense at the end of a TLV"""
Expand Down Expand Up @@ -142,10 +140,6 @@ def _maybe_calc_value(self, fieldname, otherfields):
return v
return self.calc_value(otherfields)

def val_to_bin(self, _, otherfields):
return self.underlying_type.val_to_bin(self.calc_value(otherfields),
otherfields)

def val_to_str(self, _, otherfields):
return self.underlying_type.val_to_str(self.calc_value(otherfields),
otherfields)
Expand All @@ -155,9 +149,13 @@ def name_and_val(self, name, v):
they're implied by the length of other fields"""
return ''

def val_from_bin(self, bytestream, otherfields):
def read(self, io_in, otherfields):
"""We store this, but it'll be removed from the fields as soon as it's used (i.e. by DynamicArrayType's val_from_bin)"""
return self.underlying_type.val_from_bin(bytestream, otherfields)
return self.underlying_type.read(io_in, otherfields)

def write(self, io_out, _, otherfields):
self.underlying_type.write(io_out, self.calc_value(otherfields),
otherfields)

def val_from_str(self, s):
raise ValueError('{} is implied, cannot be specified'.format(self))
Expand All @@ -182,6 +180,6 @@ def __init__(self, outer, name, elemtype, lenfield):
assert type(lenfield.fieldtype) is LengthFieldType
self.lenfield = lenfield

def val_from_bin(self, bytestream, otherfields):
return super().arr_from_bin(bytestream, otherfields,
self.lenfield.fieldtype._maybe_calc_value(self.lenfield.name, otherfields))
def read(self, io_in, otherfields):
return super().read_arr(io_in, otherfields,
self.lenfield.fieldtype._maybe_calc_value(self.lenfield.name, otherfields))
103 changes: 55 additions & 48 deletions contrib/pyln-proto/pyln/proto/message/fundamental_types.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,4 +1,22 @@
import struct
import io
from typing import Optional


def try_unpack(name: str,
io_out: io.BufferedIOBase,
structfmt: str,
empty_ok: bool) -> Optional[int]:
"""Unpack a single value using struct.unpack.

If need_all, never return None, otherwise returns None if EOF."""
b = io_out.read(struct.calcsize(structfmt))
if len(b) == 0 and empty_ok:
return None
elif len(b) < struct.calcsize(structfmt):
raise ValueError("{}: not enough bytes", name)

return struct.unpack(structfmt, b)[0]
Comment on lines +13 to +20
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

What's the advantage over the following?

try:
    struct.unpack_from(structfmt, io)
except struct.error:
    if empty_ok:
        return None
    else:
        raise

Presumably you want to avoid raising the exception at all when it can be avoided?



def split_field(s):
Expand Down Expand Up @@ -57,15 +75,11 @@ def val_from_str(self, s):
a, b = split_field(s)
return int(a), b

def val_to_bin(self, v, otherfields):
return struct.pack(self.structfmt, v)
def write(self, io_out, v, otherfields):
io_out.write(struct.pack(self.structfmt, v))

def val_from_bin(self, bytestream, otherfields):
"Returns value, bytesused"
if self.bytelen > len(bytestream):
raise ValueError('{}: not enough remaining to read'.format(self))
return struct.unpack_from(self.structfmt,
bytestream)[0], self.bytelen
def read(self, io_in, otherfields):
return try_unpack(self.name, io_in, self.structfmt, empty_ok=True)


class ShortChannelIDType(IntegerType):
Expand Down Expand Up @@ -110,30 +124,24 @@ def val_from_str(self, s):
.format(a, self.name))
return int(a), b

def val_to_bin(self, v, otherfields):
def write(self, io_out, v, otherfields):
binval = struct.pack('>Q', v)
while len(binval) != 0 and binval[0] == 0:
binval = binval[1:]
if len(binval) > self.maxbytes:
raise ValueError('{} exceeds maximum {} capacity'
.format(v, self.name))
return binval

def val_from_bin(self, bytestream, otherfields):
"Returns value, bytesused"
binval = bytes()
while len(binval) < len(bytestream):
if len(binval) == 0 and bytestream[len(binval)] == 0:
raise ValueError('{} encoding is not minimal: {}'
.format(self.name, bytestream))
binval += bytes([bytestream[len(binval)]])
io_out.write(binval)

def read(self, io_in, otherfields):
binval = io_in.read()
if len(binval) > self.maxbytes:
raise ValueError('{} is too long for {}'.format(binval, self.name))

if len(binval) > 0 and binval[0] == 0:
raise ValueError('{} encoding is not minimal: {}'
.format(self.name, binval))
# Pad with zeroes and convert as u64
return (struct.unpack_from('>Q', bytes(8 - len(binval)) + binval)[0],
len(binval))
return struct.unpack_from('>Q', bytes(8 - len(binval)) + binval)[0]


class FundamentalHexType(FieldType):
Expand All @@ -154,16 +162,18 @@ def val_from_str(self, s):
raise ValueError("Length of {} != {}", a, self.bytelen)
return ret, b

def val_to_bin(self, v, otherfields):
def write(self, io_out, v, otherfields):
if len(bytes(v)) != self.bytelen:
raise ValueError("Length of {} != {}", v, self.bytelen)
return bytes(v)
io_out.write(v)

def val_from_bin(self, bytestream, otherfields):
"Returns value, size from bytestream"
if self.bytelen > len(bytestream):
def read(self, io_in, otherfields):
val = io_in.read(self.bytelen)
if len(val) == 0:
return None
elif len(val) != self.bytelen:
raise ValueError('{}: not enough remaining'.format(self))
return bytestream[:self.bytelen], self.bytelen
return val


class BigSizeType(FieldType):
Expand All @@ -177,37 +187,34 @@ def val_from_str(self, s):

# For the convenience of TLV header parsing
@staticmethod
def to_bin(v):
def write(io_out, v, otherfields=None):
if v < 253:
return bytes([v])
io_out.write(bytes([v]))
elif v < 2**16:
return bytes([253]) + struct.pack('>H', v)
io_out.write(bytes([253]) + struct.pack('>H', v))
elif v < 2**32:
return bytes([254]) + struct.pack('>I', v)
io_out.write(bytes([254]) + struct.pack('>I', v))
else:
return bytes([255]) + struct.pack('>Q', v)
io_out.write(bytes([255]) + struct.pack('>Q', v))

@staticmethod
def from_bin(bytestream):
"Returns value, bytesused"
if bytestream[0] < 253:
return int(bytestream[0]), 1
elif bytestream[0] == 253:
return struct.unpack_from('>H', bytestream[1:])[0], 3
elif bytestream[0] == 254:
return struct.unpack_from('>I', bytestream[1:])[0], 5
def read(io_in, otherfields=None):
"Returns value, or None on EOF"
b = io_in.read(1)
if len(b) == 0:
return None
if b[0] < 253:
return int(b[0])
elif b[0] == 253:
return try_unpack('BigSize', io_in, '>H', empty_ok=False)
elif b[0] == 254:
return try_unpack('BigSize', io_in, '>I', empty_ok=False)
else:
return struct.unpack_from('>Q', bytestream[1:])[0], 9
return try_unpack('BigSize', io_in, '>Q', empty_ok=False)

def val_to_str(self, v, otherfields):
return "{}".format(int(v))

def val_to_bin(self, v, otherfields):
return self.to_bin(v)

def val_from_bin(self, bytestream, otherfields):
return self.from_bin(bytestream)


def fundamental_types():
# From 01-messaging.md#fundamental-types:
Expand Down
Loading