eip7916: add ProgressiveBitlist specs and tests

pyproject.toml

@@ -24,7 +24,7 @@ dependencies = [
   "py_arkworks_bls12381==0.3.8",
   "py_ecc==8.0.0",
   "pycryptodome==3.23.0",
-  "remerkleable @ git+https://github.com/ethereum/remerkleable@643e8e3d1d80a34f61d4b1e32a46e38ad7e57a18",
+  "remerkleable @ git+https://github.com/ethereum/remerkleable@7574640617bd87b54cd262b18ee184eba190c11e",
   "ruamel.yaml==0.18.14",
   "setuptools==80.9.0",
   "trie==3.1.0",

ssz/simple-serialize.md

@@ -16,7 +16,7 @@
   - [`uintN`](#uintn)
   - [`boolean`](#boolean)
   - [`Bitvector[N]`](#bitvectorn)
-  - [`Bitlist[N]`](#bitlistn)
+  - [`Bitlist[N]`, `ProgressiveBitlist`](#bitlistn-progressivebitlist)
   - [Vectors, containers, lists, progressive lists](#vectors-containers-lists-progressive-lists)
   - [Union](#union)
 - [Deserialization](#deserialization)
@@ -67,6 +67,9 @@
 - **bitlist**: ordered variable-length collection of `boolean` values, limited
   to `N` bits
   - notation `Bitlist[N]`
+- **progressive bitlist** _[EIP-7916, currently unused]_: ordered
+  variable-length collection of `boolean` values, without limit
+  - notation `ProgressiveBitlist`
 - **union**: union type containing one of the given subtypes
   - notation `Union[type_0, type_1, ...]`, e.g. `union[None, uint64, uint32]`
 
@@ -79,8 +82,8 @@ efficiencies.
 ### Variable-size and fixed-size
 
 We recursively define "variable-size" types to be lists, progressive lists,
-unions, `Bitlist` and all types that contain a variable-size type. All other
-types are said to be "fixed-size".
+unions, bitlists, progressive bitlists, and all composite types that contain a
+variable-size type. All other types are said to be "fixed-size".
 
 ### Byte
 
@@ -114,6 +117,7 @@ Assuming a helper function `default(type)` which returns the default value for
 | `List[type, N]`              | `[]`                                    |
 | `ProgressiveList[type]`      | `[]`                                    |
 | `Bitlist[N]`                 | `[]`                                    |
+| `ProgressiveBitlist`         | `[]`                                    |
 | `Union[type_0, type_1, ...]` | `default(type_0)`                       |
 
 #### `is_zero`
@@ -159,7 +163,7 @@ for i in range(N):
 return bytes(array)
 ```
 
-### `Bitlist[N]`
+### `Bitlist[N]`, `ProgressiveBitlist`
 
 Note that from the offset coding, the length (in bytes) of the bitlist is known.
 An additional `1` bit is added to the end, at index `e` where `e` is the length
@@ -245,10 +249,10 @@ have to do one of the following depending on what kind of object it is:
 - Containers follow the same principles as vectors, with the difference that
   there may be fixed-size objects in a container as well. This means the
   `fixed_parts` data will contain offsets as well as fixed-size objects.
-- In the case of bitlists, the length in bits cannot be uniquely inferred from
-  the number of bytes in the object. Because of this, they have a bit at the end
-  that is always set. This bit has to be used to infer the size of the bitlist
-  in bits.
+- In the case of bitlists/progressive bitlists, the length in bits cannot be
+  uniquely inferred from the number of bytes in the object. Because of this,
+  they have a bit at the end that is always set. This bit has to be used to
+  infer the size of the bitlist in bits.
 - In the case of unions, the first byte of the deserialization scope is
   deserialized as type selector, the remainder of the scope is deserialized as
   the selected type.
@@ -333,6 +337,8 @@ recursively:
   a progressive list of basic objects.
 - `mix_in_length(merkleize(pack_bits(value), limit=chunk_count(type)), len(value))`
   if `value` is a bitlist.
+- `mix_in_length(merkleize_progressive(pack_bits(value)), len(value))` if
+  `value` is a progressive bitlist.
 - `merkleize([hash_tree_root(element) for element in value])` if `value` is a
   vector of composite objects or a container.
 - `mix_in_length(merkleize([hash_tree_root(element) for element in value], limit=chunk_count(type)), len(value))`
@@ -385,6 +391,7 @@ value. Parsers may ignore additional JSON fields.
 | `ProgressiveList[type]`      | array           | `[element, ...]`                         |
 | `ProgressiveList[byte]`      | hex-byte-string | `"0x1122"`                               |
 | `Bitlist[N]`                 | hex-byte-string | `"0x1122"`                               |
+| `ProgressiveBitlist`         | hex-byte-string | `"0x1122"`                               |
 | `Union[type_0, type_1, ...]` | selector-object | `{ "selector": number, "data": type_N }` |
 
 Integers are encoded as strings to avoid loss of precision in 64-bit values.
@@ -395,8 +402,8 @@ Aliases are encoded as their underlying type.
 appear in an SSZ stream.
 
 `List`, `ProgressiveList`, and `Vector` of `byte` (and aliases thereof) are
-encoded as `hex-byte-string`. `Bitlist` and `Bitvector` similarly map their
-SSZ-byte encodings to a `hex-byte-string`.
+encoded as `hex-byte-string`. `Bitlist`, `ProgressiveBitlist`, and `Bitvector`
+similarly map their SSZ-byte encodings to a `hex-byte-string`.
 
 `Union` is encoded as an object with a `selector` and `data` field, where the
 contents of `data` change according to the selector.

tests/core/pyspec/eth2spec/debug/encode.py

@@ -5,6 +5,7 @@
     boolean,
     Container,
     List,
+    ProgressiveBitlist,
     ProgressiveList,
     uint,
     Union,
@@ -20,7 +21,7 @@ def encode(value, include_hash_tree_roots=False):
         return int(value)
     elif isinstance(value, boolean):
         return value == 1
-    elif isinstance(value, Bitlist | Bitvector):
+    elif isinstance(value, Bitlist | ProgressiveBitlist | Bitvector):
         return "0x" + serialize(value).hex()
     elif isinstance(value, list):  # normal python lists
         return [encode(element, include_hash_tree_roots) for element in value]

tests/core/pyspec/eth2spec/debug/random_value.py

@@ -10,6 +10,7 @@
     ByteVector,
     Container,
     List,
+    ProgressiveBitlist,
     ProgressiveList,
     uint,
     Union,
@@ -103,10 +104,10 @@ def get_random_ssz_object(
             get_random_ssz_object(rng, elem_type, max_bytes_length, max_list_length, mode, chaos)
             for _ in range(typ.vector_length())
         )
-    elif issubclass(typ, List) or issubclass(typ, ProgressiveList) or issubclass(typ, Bitlist):
+    elif issubclass(typ, List | ProgressiveList | Bitlist | ProgressiveBitlist):
         limit = max_list_length
         # SSZ imposes a hard limit on lists, we can't put in more than that
-        if not issubclass(typ, ProgressiveList) and typ.limit() < limit:
+        if not issubclass(typ, ProgressiveList | ProgressiveBitlist) and typ.limit() < limit:
             limit = typ.limit()
 
         length = rng.randint(0, limit)
@@ -117,7 +118,7 @@ def get_random_ssz_object(
         elif mode == RandomizationMode.mode_nil_count:
             length = 0
 
-        elem_type = boolean if issubclass(typ, Bitlist) else typ.element_cls()
+        elem_type = boolean if issubclass(typ, Bitlist | ProgressiveBitlist) else typ.element_cls()
         max_list_length = 1 << (max_list_length.bit_length() >> 1)
         return typ(
             get_random_ssz_object(rng, elem_type, max_bytes_length, max_list_length, mode, chaos)

tests/core/pyspec/eth2spec/utils/ssz/ssz_typing.py

@@ -25,7 +25,7 @@
 )
 from remerkleable.complex import Container, List, Vector
 from remerkleable.core import BasicView, Path, View
-from remerkleable.progressive import ProgressiveList
+from remerkleable.progressive import ProgressiveBitlist, ProgressiveList
 from remerkleable.union import Union
 
 Bytes20 = ByteVector[20]  # type: ignore

tests/formats/ssz_generic/README.md

@@ -22,6 +22,8 @@ into a SSZ type:
 - Bitfields
   - `bitvector`
   - `bitlist`
+- ProgressiveBitlist
+  - `progressive_bitlist`
 - Basic types
   - `boolean`
   - `uints`
@@ -144,6 +146,14 @@ Data:
 {length}: the length, in bits, of the bitvector.
 ```
 
+### `progressive_bitlist`
+
+```
+Template:
+
+progbitlist
+```
+
 ### `boolean`
 
 A boolean has no type variations. Instead, file names just plainly describe the
@@ -221,4 +231,19 @@ class BitsStruct(Container):
     C: Bitvector[1]
     D: Bitlist[6]
     E: Bitvector[8]
+
+
+class ProgressiveBitsStruct(Container):
+    A: Bitvector[256]
+    B: Bitlist[256]
+    C: ProgressiveBitlist
+    D: Bitvector[257]
+    E: Bitlist[257]
+    F: ProgressiveBitlist
+    G: Bitvector[1280]
+    H: Bitlist[1280]
+    I: ProgressiveBitlist
+    J: Bitvector[1281]
+    K: Bitlist[1281]
+    L: ProgressiveBitlist
 ```

tests/generators/runners/ssz_generic.py

@@ -10,6 +10,7 @@
     ssz_bitvector,
     ssz_boolean,
     ssz_container,
+    ssz_progressive_bitlist,
     ssz_uints,
 )
 
@@ -26,10 +27,12 @@ def get_test_cases() -> Iterable[TestCase]:
         ("bitvector", "invalid", ssz_bitvector.invalid_cases),
         ("boolean", "valid", ssz_boolean.valid_cases),
         ("boolean", "invalid", ssz_boolean.invalid_cases),
-        ("uints", "valid", ssz_uints.valid_cases),
-        ("uints", "invalid", ssz_uints.invalid_cases),
         ("containers", "valid", ssz_container.valid_cases),
         ("containers", "invalid", ssz_container.invalid_cases),
+        ("progressive_bitlist", "valid", ssz_progressive_bitlist.valid_cases),
+        ("progressive_bitlist", "invalid", ssz_progressive_bitlist.invalid_cases),
+        ("uints", "valid", ssz_uints.valid_cases),
+        ("uints", "invalid", ssz_uints.invalid_cases),
     ]
 
     test_cases = []

tests/generators/runners/ssz_generic_cases/ssz_bitlist.py

@@ -6,21 +6,32 @@
 
 from .ssz_test_case import invalid_test_case, valid_test_case
 
+INVALID_BITLIST_CASES = [
+    ("no_delimiter_empty", b""),
+    ("no_delimiter_zero_byte", b"\x00"),
+    ("no_delimiter_zeroes", b"\x00\x00\x00"),
+]
 
-def bitlist_case_fn(rng: Random, mode: RandomizationMode, limit: int):
-    return get_random_ssz_object(
+
+def bitlist_case_fn(
+    rng: Random, mode: RandomizationMode, limit: int, force_final_bit: bool | None = None
+):
+    bits = get_random_ssz_object(
         rng,
         Bitlist[limit],
         max_bytes_length=(limit // 8) + 1,
         max_list_length=limit,
         mode=mode,
         chaos=False,
     )
+    if force_final_bit is not None and bits.length() > 0:
+        bits[bits.length() - 1] = force_final_bit
+    return bits
 
 
 def valid_cases():
     rng = Random(1234)
-    for size in [1, 2, 3, 4, 5, 8, 16, 31, 512, 513]:
+    for size in [1, 2, 3, 4, 5, 6, 7, 8, 9, 15, 16, 17, 31, 32, 33, 511, 512, 513]:
         for variation in range(5):
             for mode in [
                 RandomizationMode.mode_nil_count,
@@ -32,24 +43,28 @@ def valid_cases():
                 yield (
                     f"bitlist_{size}_{mode.to_name()}_{variation}",
                     valid_test_case(
-                        lambda rng=rng, mode=mode, size=size: bitlist_case_fn(rng, mode, size)
+                        lambda rng=rng, mode=mode, size=size, variation=variation: bitlist_case_fn(
+                            rng, mode, size, force_final_bit=[None, True, False][variation % 3]
+                        )
                     ),
                 )
 
 
 def invalid_cases():
-    yield "bitlist_no_delimiter_empty", invalid_test_case(lambda: b"")
-    yield "bitlist_no_delimiter_zero_byte", invalid_test_case(lambda: b"\x00")
-    yield "bitlist_no_delimiter_zeroes", invalid_test_case(lambda: b"\x00\x00\x00")
+    for description, data in INVALID_BITLIST_CASES:
+        yield f"bitlist_{description}", invalid_test_case(lambda data=data: data)
     rng = Random(1234)
     for typ_limit, test_limit in [
         (1, 2),
+        (1, 7),
         (1, 8),
         (1, 9),
         (2, 3),
         (3, 4),
         (4, 5),
         (5, 6),
+        (6, 7),
+        (7, 8),
         (8, 9),
         (32, 64),
         (32, 33),

tests/generators/runners/ssz_generic_cases/ssz_bitvector.py

@@ -30,7 +30,7 @@ def bitvector_case_fn(
 
 def valid_cases():
     rng = Random(1234)
-    for size in [1, 2, 3, 4, 5, 8, 16, 31, 512, 513]:
+    for size in [1, 2, 3, 4, 5, 6, 7, 8, 9, 15, 16, 17, 31, 32, 33, 511, 512, 513]:
         for mode in [
             RandomizationMode.mode_random,
             RandomizationMode.mode_zero,