Skip to content

Commit e62bb6c

Browse files
samuelsamuel
committed
initial commit
0 parents  commit e62bb6c

4 files changed

Lines changed: 234 additions & 0 deletions

File tree

‎.gitignore‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
*.pyc
2+
.DS_Store

‎erlastic/__init__.py‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,10 @@
1+
2+
"""Erlang External Term Format serializer/deserializer"""
3+
4+
from erlastic.codec import ErlangTermDecoder, ErlangTermEncoder, Atom, Binary
5+
6+
def encode(obj):
7+
return ErlangTermEncoder().encode(obj)
8+
9+
def decode(obj):
10+
return ErlangTermDecoder().decode(obj)

‎erlastic/codec.py‎

Lines changed: 172 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,172 @@
1+
2+
from __future__ import division
3+
4+
import math
5+
import struct
6+
7+
NEW_FLOAT_EXT = 'F' # 70 [Float64:IEEE float]
8+
SMALL_INTEGER_EXT = 'a' # 97 [UInt8:Int] Unsigned 8 bit integer
9+
INTEGER_EXT = 'b' # 98 [Int32:Int] Signed 32 bit integer in big-endian format
10+
FLOAT_EXT = 'c' # 99 [31:Float String] Float in string format (formatted "%.20e", sscanf "%lf"). Superseded by NEW_FLOAT_EXT
11+
ATOM_EXT = 'd' # 100 [UInt16:Len, Len:AtomName] max Len is 255
12+
SMALL_TUPLE_EXT = 'h' # 104 [UInt8:Arity, N:Elements]
13+
LARGE_TUPLE_EXT = 'i' # 105 [UInt32:Arity, N:Elements]
14+
NIL_EXT = 'j' # 106 empty list
15+
STRING_EXT = 'k' # 107 [UInt32:Len, Len:Characters]
16+
LIST_EXT = 'l' # 108 [UInt32:Len, Elements, Tail]
17+
BINARY_EXT = 'm' # 109 [UInt32:Len, Len:Data]
18+
SMALL_BIG_EXT = 'n' # 110 [UInt8:n, UInt8:Sign, n:nums]
19+
LARGE_BIG_EXT = 'o' # 111 [UInt32:n, UInt8:Sign, n:nums]
20+
21+
class Atom(str):
22+
def __repr__(self):
23+
return "Atom(%s)" % super(Atom, self).__repr__()
24+
25+
class Binary(str):
26+
def __repr__(self):
27+
return "Binary(%s)" % super(Binary, self).__repr__()
28+
29+
class ErlangTermDecoder(object):
30+
def __init__(self):
31+
pass
32+
33+
def decode(self, bytes, offset=0):
34+
if bytes[offset] == "\x83": # Version 131
35+
offset += 1
36+
return self._decode(bytes, offset)[0]
37+
38+
def _decode(self, bytes, offset=0):
39+
tag = bytes[offset]
40+
offset += 1
41+
if tag == SMALL_INTEGER_EXT:
42+
return ord(bytes[offset]), offset+1
43+
elif tag == INTEGER_EXT:
44+
return struct.unpack(">l", bytes[offset:offset+4])[0], offset+4
45+
elif tag == FLOAT_EXT:
46+
return float(bytes[offset:offset+31].split('\x00', 1)[0]), offset+31
47+
elif tag == NEW_FLOAT_EXT:
48+
return struct.unpack(">d", bytes[offset:offset+8])[0], offset+8
49+
elif tag == ATOM_EXT:
50+
atom_len = struct.unpack(">H", bytes[offset:offset+2])[0]
51+
atom = bytes[offset+2:offset+2+atom_len]
52+
offset += 2+atom_len
53+
if atom == "true":
54+
return True, offset
55+
elif atom == "false":
56+
return False, offset
57+
return Atom(atom), offset
58+
elif tag in (SMALL_TUPLE_EXT, LARGE_TUPLE_EXT):
59+
if tag == SMALL_TUPLE_EXT:
60+
arity = ord(bytes[offset])
61+
offset += 1
62+
else:
63+
arity = struct.unpack(">L", bytes[offset:offset+4])[0]
64+
offset += 4
65+
66+
items = []
67+
for i in range(arity):
68+
val, offset = self._decode(bytes, offset)
69+
items.append(val)
70+
return tuple(items), offset
71+
elif tag == NIL_EXT:
72+
return [], offset
73+
elif tag == STRING_EXT:
74+
length = struct.unpack(">H", bytes[offset:offset+2])[0]
75+
return bytes[offset+2:offset+2+length], offset+2+length
76+
elif tag == LIST_EXT:
77+
length = struct.unpack(">L", bytes[offset:offset+4])[0]
78+
offset += 4
79+
items = []
80+
for i in range(length):
81+
val, offset = self._decode(bytes, offset)
82+
items.append(val)
83+
tail, offset = self._decode(bytes, offset)
84+
if tail != []:
85+
# TODO: Not sure what to do with the tail
86+
raise NotImplementedError("Lists with non empty tails are not supported")
87+
return items, offset
88+
elif tag == BINARY_EXT:
89+
length = struct.unpack(">L", bytes[offset:offset+4])[0]
90+
return Binary(bytes[offset+4:offset+4+length]), offset+4+length
91+
elif tag in (SMALL_BIG_EXT, LARGE_BIG_EXT):
92+
if tag == SMALL_BIG_EXT:
93+
n = ord(bytes[offset])
94+
offset += 1
95+
else:
96+
n = struct.unpack(">L", bytes[offset:offset+4])[0]
97+
offset += 4
98+
sign = ord(bytes[offset])
99+
offset += 1
100+
b = 1
101+
val = 0
102+
for i in range(n):
103+
val += ord(bytes[offset]) * b
104+
b <<= 8
105+
offset += 1
106+
if sign != 0:
107+
val = -val
108+
return val, offset
109+
else:
110+
raise NotImplementedError("Unsupported tag %d" % tag)
111+
112+
class ErlangTermEncoder(object):
113+
def __init__(self):
114+
pass
115+
116+
def encode(self, obj):
117+
bytes = [chr(131)]
118+
self._encode(obj, bytes)
119+
return "".join(bytes)
120+
121+
def _encode(self, obj, bytes):
122+
if obj is False:
123+
bytes += [ATOM_EXT, struct.pack(">H", 5), "false"]
124+
elif obj is True:
125+
bytes += [ATOM_EXT, struct.pack(">H", 4), "true"]
126+
elif isinstance(obj, (int, long)):
127+
if 0 <= obj <= 255:
128+
bytes += [SMALL_INTEGER_EXT, chr(obj)]
129+
elif -2147483648 <= obj <= 2147483647:
130+
bytes += [INTEGER_EXT, struct.pack(">l", obj)]
131+
else:
132+
n = int(math.ceil(math.log(obj, 2) / 8))
133+
if n <= 256:
134+
bytes += [SMALL_BIG_EXT, chr(n)]
135+
else:
136+
bytes += [LARGE_BIG_EXT, struct.pack(">L", n)]
137+
if obj >= 0:
138+
bytes.append("\x00")
139+
else:
140+
bytes.append("\x01")
141+
obj = -obj
142+
while obj > 0:
143+
bytes.append(chr(obj & 0xff))
144+
obj >>= 8
145+
elif isinstance(obj, float):
146+
floatstr = "%.20e" % obj
147+
bytes += [FLOAT_EXT, floatstr + "\x00"*(31-len(floatstr))]
148+
elif isinstance(obj, Atom):
149+
bytes += [ATOM_EXT, struct.pack(">H", len(obj)), obj]
150+
elif isinstance(obj, (Binary, buffer)):
151+
bytes += [BINARY_EXT, struct.pack(">L", len(obj)), obj]
152+
elif isinstance(obj, str):
153+
bytes += [STRING_EXT, struct.pack(">H", len(obj)), obj]
154+
elif isinstance(obj, unicode):
155+
self._encode([ord(x) for x in obj], bytes)
156+
elif isinstance(obj, tuple):
157+
n = len(obj)
158+
if n < 256:
159+
bytes += [SMALL_TUPLE_EXT, chr(n)]
160+
else:
161+
bytes += [LARGE_TUPLE_EXT, struct.pack(">L", n)]
162+
for item in obj:
163+
self._encode(item, bytes)
164+
elif obj == []:
165+
bytes.append(NIL_EXT)
166+
elif isinstance(obj, list):
167+
bytes += [LIST_EXT, struct.pack(">L", len(obj))]
168+
for item in obj:
169+
self._encode(item, bytes)
170+
bytes.append(NIL_EXT) # list tail - no such thing in Python
171+
else:
172+
raise NotImplementedError("Unable to serialize %r" % obj)

‎tests.py‎

Lines changed: 50 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,50 @@
1+
#!/usr/bin/env python
2+
3+
import unittest
4+
5+
from erlastic import decode, encode, Atom, Binary
6+
7+
erlang_term_binaries = [
8+
# nil
9+
([], list, "\x83j"),
10+
# binary
11+
(Binary("foo"), Binary, '\x83m\x00\x00\x00\x03foo'),
12+
# atom
13+
(Atom("foo"), Atom, '\x83d\x00\x03foo'),
14+
# atom true
15+
(True, bool, '\x83d\x00\x04true'),
16+
# atom false
17+
(False, bool, '\x83d\x00\x05false'),
18+
# string
19+
("foo", str, '\x83k\x00\x03foo'),
20+
# small integer
21+
(123, int, '\x83a{'),
22+
# integer
23+
(12345, int, '\x83b\x00\x0009'),
24+
# float
25+
(1.2345, float, '\x83c1.23449999999999993072e+00\x00\x00\x00\x00\x00'),
26+
# tuple
27+
((Atom("foo"), "test", 123), tuple, '\x83h\x03d\x00\x03fook\x00\x04testa{'),
28+
# list
29+
([1024, "test", 4.096], list, '\x83l\x00\x00\x00\x03b\x00\x00\x04\x00k\x00\x04testc4.09600000000000008527e+00\x00\x00\x00\x00\x00j'),
30+
# small big
31+
(12345678901234567890, long, '\x83n\x08\x00\xd2\n\x1f\xeb\x8c\xa9T\xab'),
32+
# large big
33+
(123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890,
34+
long, '\x83o\x00\x00\x01D\x00\xd2\n?\xce\x96\xf1\xcf\xacK\xf1{\xefa\x11=$^\x93\xa9\x88\x17\xa0\xc2\x01\xa5%\xb7\xe3Q\x1b\x00\xeb\xe7\xe5\xd5Po\x98\xbd\x90\xf1\xc3\xddR\x83\xd1)\xfc&\xeaH\xc31w\xf1\x07\xf3\xf33\x8f\xb7\x96\x83\x05t\xeci\x9cY"\x98\x98i\xca\x11bY=\xcc\xa1\xb4R\x1bl\x01\x86\x18\xe9\xa23\xaa\x14\xef\x11[}O\x14RU\x18$\xfe\x7f\x96\x94\xcer?\xd7\x8b\x9a\xa7v\xbd\xbb+\x07X\x94x\x7fI\x024.\xa0\xcc\xde\xef:\xa7\x89~\xa4\xafb\xe4\xc1\x07\x1d\xf3cl|0\xc9P`\xbf\xab\x95z\xa2DQf\xf7\xca\xef\xb0\xc4=\x11\x06*:Y\xf58\xaf\x18\xa7\x81\x13\xdf\xbdTl4\xe0\x00\xee\x93\xd6\x83V\xc9<\xe7I\xdf\xa8.\xf5\xfc\xa4$R\x95\xef\xd1\xa7\xd2\x89\xceu!\xf8\x08\xb1Zv\xa6\xd9z\xdb0\x88\x10\xf3\x7f\xd3sc\x98[\x1a\xac6V\x1f\xad0)\xd0\x978\xd1\x02\xe6\xfbH\x149\xdc).\xb5\x92\xf6\x91A\x1b\xcd\xb8`B\xc6\x04\x83L\xc0\xb8\xafN+\x81\xed\xec?;\x1f\xab1\xc1^J\xffO\x1e\x01\x87H\x0f.ZD\x06\xf0\xbak\xaagVH]\x17\xe6I.B\x14a2\xc1;\xd1+\xea.\xe4\x92\x15\x93\xe9\'E\xd0(\xcd\x90\xfb\x10'),
35+
]
36+
37+
class ErlangTestCase(unittest.TestCase):
38+
def testDecode(self):
39+
for python, expected_type, erlang in erlang_term_binaries:
40+
decoded = decode(erlang)
41+
self.failUnlessEqual(python, decoded)
42+
self.failUnless(isinstance(decoded, expected_type))
43+
44+
def testEncode(self):
45+
for python, expected_type, erlang in erlang_term_binaries:
46+
encoded = encode(python)
47+
self.failUnlessEqual(erlang, encoded)
48+
49+
if __name__ == '__main__':
50+
unittest.main()

0 commit comments

Comments
 (0)