feat: implement new binary format, BITE

Author: bitwalker

CHANGELOG.md

@@ -32,6 +32,8 @@
 - Added `multicall` support for the CLI ([#1141](https://github.com/0xMiden/miden-vm/pull/2081))
 - Made `miden-prover`'s metal prover async-compatible. ([#2133](https://github.com/0xMiden/miden-vm/pull/2133)).
 - Abstract away the fast processor's operation execution into a new `Processor` trait ([#2141](https://github.com/0xMiden/miden-vm/pull/2141))
+- Added ability to serialize packages using BITE ([#2071](https://github.com/0xMiden/miden-vm/pull/2071))
+- Implemented new binary serialization format, BITE ([#2071](https://github.com/0xMiden/miden-vm/pull/2071))
 
 ## 0.17.1 (2025-08-29)
 

Cargo.lock

@@ -59,9 +59,8 @@ miden-formatting = { version = "0.1", default-features = false }
 midenc-hir-type = { version = "0.1", default-features = false }
 
 # Third-party crates
-insta = { version = "1.43", default-features = false, features = [
-    "colors",
-] }
+env_logger = "0.11"
+insta = { version = "1.43", default-features = false, features = ["colors"] }
 log = { version = "0.4", default-features = false }
 paste = { version = "1.0", default-features = false }
 proptest = { version = "1.7", default-features = false, features = [

Cargo.toml

@@ -30,7 +30,7 @@ logging = ["dep:env_logger"]
 
 [dependencies]
 aho-corasick = { version = "1.1", default-features = false }
-env_logger = { version = "0.11", optional = true }
+env_logger = { workspace = true, optional = true }
 lalrpop-util = { version = "0.22", default-features = false }
 log.workspace = true
 miden-core.workspace = true

assembly-syntax/Cargo.toml

@@ -24,7 +24,7 @@ testing = ["logging", "miden-assembly-syntax/testing"]
 logging = ["dep:env_logger"]
 
 [dependencies]
-env_logger = { version = "0.11", optional = true }
+env_logger = { workspace = true, optional = true }
 log.workspace = true
 miden-assembly-syntax.workspace = true
 miden-core.workspace = true

assembly/Cargo.toml

@@ -0,0 +1,31 @@
+[package]
+name = "miden-bite"
+version = "0.18.0"
+description = "BITE is Binary Interchange, Tiny Encoding - a compact binary file format"
+documentation = "https://docs.rs/miden-bite/0.16.0"
+readme = "README.md"
+categories = ["no-std"]
+edition.workspace = true
+rust-version.workspace = true
+license.workspace = true
+authors.workspace = true
+homepage.workspace = true
+repository.workspace = true
+exclude.workspace = true
+
+[features]
+default = ["std"]
+std = ["indexmap/std", "serde/std", "thiserror/std"]
+
+[dependencies]
+bumpalo = { version = "3.19", default-features = false }
+log.workspace = true
+indexmap = { version = "2.10", default-features = false }
+rustc-hash = { version = "2.1", default-features = false }
+serde.workspace = true
+smallvec.workspace = true
+thiserror.workspace = true
+
+[dev-dependencies]
+env_logger.workspace = true
+proptest.workspace = true

crates/utils/bite/Cargo.toml

@@ -0,0 +1,28 @@
+# miden-bite
+
+This crate implements a `serde`-compatible binary encoding which we call BITE, which stands for _Binary Interchange, Tiny Encoding_. As the name implies, it is designed for exchanging data in binary form efficiently - in particular, we aim to use this as the underlying encoding for the Miden package format, and similar use cases.
+
+## Design principles
+
+BITE is designed with the following properties in mind:
+
+* Compact
+* Versioned
+* Validatable, i.e. the ability to validate the structure of the input without
+  needing to deserialize it.
+* Use `serde`'s data model to allow for encoding arbitrary Rust data types
+* Enable re-use of `serde`'s `Serialize` and `Deserialize` trait impls for both human-readable and BITE-encoded formats.
+* Minimally self-describing to allow for supporting `serde` features which require this, e.g. conditionally-skipped fields
+
+## Size reduction techniques
+
+The following are the techniques we use to acheive the goal of compact encoding of arbitrary Rust data structures:
+
+* _Transparent string interning_. This de-duplicates strings that are encoded when serializing a given data structure, storing only a single copy of each unique string, assigning each an integer identifier, and then storing only the identifier at each place the string is used. For structures with many copies of the same string (e.g. an AST where each node holds a reference to the file and line it corresponds to at the source level), this vastly reduces the size of the encoded binary. If all strings in the input data structure are already unique, interning does introduce some minimal overhead; but for our use cases, duplication is far more common than not, and this technique is always beneficial.
+* _All integers use variable-length encoding_. Statisically, most integer values are, in practice, small values. Encoding such values using the number of bits equivalent to their maximum possible value is immensely wasteful. Consider using a `u32` to represent an index into an array, where all instances of that array are going to be smaller than 256 elements - encoding this value as a `u32` is going to waste 3 bytes for every index. Instead, using a variable-length encoding, the vast majority of indices will only require a single byte, with indices in the range 129-256 requiring a second byte. These savings add up considerably considering how frequently integer values are encoded.
+* _Booleans are intrinsically-tagged_. In other words, the encoding of `true` and `false`, as `1` and `0` respectively, are also valid type tags for the boolean type. So storing a boolean requires only a single byte for both type tag and value, rather than a byte for each.
+* _`Option<T>` is intrinsically tagged_. Similar to booleans, the value for `None` is just the type tag for `None`, while `Some` is encoded as the tag for `Some` followed by the encoded value of type `T`.
+
+## Structure
+
+The structure of a BITE-encoded stream is described using the Kaitai Struct language in [bite.ksy].

crates/utils/bite/README.md

@@ -0,0 +1,170 @@
+meta:
+  id: bite
+  title: BITE-encoded file
+  file-extension: bite
+  endian: le
+  bit-endian: le
+seq:
+  - id: magic
+    contents: 'BITE\0'
+  - id: version
+    type: u8
+  - id: strings
+    type: string_table
+  - id: payload
+    type: payload
+    repeat: eos
+enum:
+  tag:
+    false: 0
+    true: 1
+    none: 2
+    some: 3
+    int: 4
+    sint: 5
+    i8: 6
+    u8: 7
+    f32: 8
+    f64: 9
+    char: 10
+    bytes: 11
+    str: 12
+    seq: 13
+    map: 14
+    unit_variant: 15
+    newtype_variant: 16
+    struct_variant: 17
+    tuple_variant: 18
+types:
+  unit:
+    seq:
+      - id: empty
+        size: 0
+  payload:
+    doc: An encoded value tagged with its serde data type
+    seq:
+      - id: tag
+        type: u8
+        enum: tag
+      - id: value
+        type:
+          switch-on: tag
+          cases:
+            'tag::false': unit
+            'tag::true': unit
+            'tag::none': unit
+            'tag::some': payload
+            'tag::int': varint
+            'tag::sint': varint
+            'tag::i8': u8
+            'tag::u8': u8
+            'tag::f32': f32
+            'tag::f64': f64
+            'tag::char': varint
+            'tag::bytes': payload_bytes
+            'tag::str': varint
+            'tag::seq': payload_seq
+            'tag::map': payload_map
+            'tag::unit_variant': unit
+            'tag::newtype_variant': payload
+            'tag::struct_variant': payload_variant
+            'tag::tuple_variant': payload_variant
+  payload_variant:
+    doc: A hint to the deserializer that a given enum variant is next in the stream
+    seq:
+      - id: variant_id
+        type: varint
+  payload_seq:
+    seq:
+      - id: num_elements
+        type: varint
+      - id: num_element_bytes
+        type: varint
+        if: num_elements.value > 0
+        doc: The number of subsequent bytes holding the encoded elements of this sequence
+      - id: elements
+        type: payload
+        if: num_elements.value > 0
+        size: num_element_bytes.value
+        repeat: expr
+        repeat-expr: num_elements.value
+  payload_map:
+    seq:
+      - id: num_elements
+        type: varint
+      - id: num_key_bytes
+        type: varint
+        if: num_elements.value > 0
+        doc: The number of subsequent bytes holding the encoded keys of this map
+      - id: keys
+        type: payload
+        if: num_elements.value > 0
+        size: num_key_bytes.value
+        repeat: expr
+        repeat-expr: num_elements.value
+      - id: num_value_bytes
+        type: varint
+        if: num_elements.value > 0
+        doc: The number of subsequent bytes holding the encoded values of this map
+      - id: values
+        type: payload
+        if: num_elements.value > 0
+        size: num_value_bytes.value
+        repeat: expr
+        repeat-expr: num_elements.value
+  payload_bytes:
+    seq:
+      - id: num_bytes
+        type: varint
+      - id: bytes
+        size: num_bytes.value
+  strings_table:
+    doc: The interned strings table
+    seq:
+      - id: num_strings
+        type: varint
+      - id: string_entries
+        type: string_entry
+        repeat: expr
+        repeat-expr: num_strings.value
+  string_entry:
+    doc: An entry in the interned string table
+    seq:
+      - id: string_len
+        type: varint
+      - id: string_data
+        type: str
+        size: string_len.value
+        encoding: UTF-8
+  varint:
+    doc: A variable-length encoded integer value
+    seq:
+      - id: varint_groups
+        type: varint_group
+        repeat: until
+        repeat-until: not _.has_next
+    instances:
+      last:
+        value: varint_groups.size - 1
+      value:
+        value: >-
+          groups[last].value
+          + (last >= 1 ? (varint_groups[last - 1].value << 7) : 0)
+          + (last >= 2 ? (varint_groups[last - 2].value << 14) : 0)
+          + (last >= 3 ? (varint_groups[last - 3].value << 21) : 0)
+          + (last >= 4 ? (varint_groups[last - 4].value << 28) : 0)
+          + (last >= 5 ? (varint_groups[last - 5].value << 35) : 0)
+          + (last >= 6 ? (varint_groups[last - 6].value << 42) : 0)
+          + (last >= 7 ? (varint_groups[last - 7].value << 49) : 0)
+        doc: Resulting value as normal integer
+  varint_group:
+    seq:
+      - id: b
+        type: u1
+    instances:
+      has_next:
+        value: (b & 0b1000_0000) != 0
+        doc: If true, then we have more bytes to read
+      value:
+        value: b & 0b0111_1111
+        doc: The 7-bit (base128) numeric value chunk of this group