feat: #84: add APP11 segment division logic in case of overflow

README.md

@@ -8,9 +8,9 @@
 
 [![Linting](https://github.com/TourmalineCore/c2pie/actions/workflows/lint-on-pull-request.yml/badge.svg?branch=develop)](https://github.com/TourmalineCore/c2pie/actions/workflows/lint-on-pull-request.yml)
 [![c2pa](https://img.shields.io/badge/c2pa-v1.4-seagreen.svg)](https://c2pa.org/)
-[![coverage](https://img.shields.io/badge/e2e_coverage-69.63%25-orange)](https://github.com/TourmalineCore/c2pie/actions/workflows/calculate-tests-coverage-on-pull-request.yml)
-[![coverage](https://img.shields.io/badge/units_coverage-86.82%25-olivedrab)](https://github.com/TourmalineCore/c2pie/actions/workflows/calculate-tests-coverage-on-pull-request.yml)
-[![coverage](https://img.shields.io/badge/full_coverage-91.89%25-forestgreen)](https://github.com/TourmalineCore/c2pie/actions/workflows/calculate-tests-coverage-on-pull-request.yml)
+[![coverage](https://img.shields.io/badge/e2e_coverage-68.80%25-orange)](https://github.com/TourmalineCore/c2pie/actions/workflows/calculate-tests-coverage-on-pull-request.yml)
+[![coverage](https://img.shields.io/badge/units_coverage-86.30%25-olivedrab)](https://github.com/TourmalineCore/c2pie/actions/workflows/calculate-tests-coverage-on-pull-request.yml)
+[![coverage](https://img.shields.io/badge/full_coverage-91.35%25-forestgreen)](https://github.com/TourmalineCore/c2pie/actions/workflows/calculate-tests-coverage-on-pull-request.yml)
 [![latest](https://img.shields.io/pypi/v/c2pie?label=latest&colorB=fc8021)](https://pypi.org/project/c2pie/)
 
 <br>

c2pie/c2pa/manifest_store.py

@@ -38,4 +38,10 @@ def add_full_c2pa_structure_exclusion(
         super().sync_payload()
 
     def serialize(self) -> bytes:
+        if self.l_box > 0xFFFFFFFF:
+            raise ValueError(
+                f"Manifest Store is too large to serialize: {self.l_box:,} bytes. "
+                "The JUMBF LBox field is limited to 4 bytes (max 4,294,967,295 bytes)."
+            )
+
         return super().serialize()

c2pie/c2pa_injection/jpg_injection.py

@@ -6,9 +6,7 @@
 #   - 2 (bytes of CI)
 #   - 2 (bytes of EN)
 #   - 4 (bytes of Z)
-#   - 4 (bytes of LBox)
-#   - 4 (bytes of TBox)
-JPG_SEGMENT_MAX_PAYLOAD_LENGTH = 65517
+JPG_SEGMENT_MAX_PAYLOAD_LENGTH = 65525
 
 
 class JpgSegment:
@@ -21,7 +19,7 @@ def __init__(
         self.payload_length = payload_length
 
     def get_segment_length(self):
-        return self.payload_length + 2  # payload length + size of payload length
+        return self.payload_length + 2  # payload length + size of marker bytes length
 
     def serialize(
         self,
@@ -37,69 +35,93 @@ def serialize(
 
 
 class JpgSegmentApp11(JpgSegment):
-    def __init__(self, segment_id, sequence_number, payload_length, payload):
+    def __init__(
+        self,
+        segment_id,
+        sequence_number,
+        payload_length,
+        payload,
+    ):
         self.ci = bytes.fromhex(b"JP".hex())  # 2 bytes
-        self.en = segment_id  # 2 bytes
-        self.z = sequence_number  # 4 bytes
+        self.en = segment_id.to_bytes(2, "big")  # 2 bytes
+        self.z = sequence_number.to_bytes(4, "big")  # 4 bytes
 
         self.app11_payload = payload
 
-        super().__init__(payload_length=self.get_payload_length(payload_length))
-
-    def get_payload_length(self, payload_length):
-        return 2 + 2 + 4 + 4 + 4 + payload_length
+        super().__init__(
+            payload_length=self.get_payload_length(payload_length),
+        )
 
-    def serialize(self, payload=None):
-        _en = self.en.to_bytes(2, "big")
-        _z = self.z.to_bytes(4, "big")
+    def get_payload_length(
+        self,
+        payload_length,
+    ):
+        return 2 + 2 + 4 + payload_length
 
-        app11_payload = self.ci + _en + _z + self.app11_payload
+    def serialize(
+        self,
+        payload=None,
+    ):
+        app11_payload = self.ci + self.en + self.z + self.app11_payload
 
         return super().serialize(app11_payload)
 
 
 class JpgSegmentApp11Storage:
     def __init__(
         self,
-        app11_segment_box_length: int,
-        app11_segment_box_type: str,
         payload: bytes,
     ):
-        self.l_box = app11_segment_box_length
-        self.t_box = app11_segment_box_type
         self.payload = payload
         self.serialized_length = 0
 
-    def get_payload_length(self):
-        return self.l_box - 4 - 4  # - t_box_length - l_box_length
-
     def get_serialized_length(self):
         return self.serialized_length
 
     def serialize(self):
-        segment_id = 1
+        # CI: JPEG extensions marker - JP.
+        # EN: Box instance number.
+        # Z: Packet sequence number.
+
+        # TODO: It must not conflict with any other identifiers, if they exist.
+        # We choose it randomly, but we will need to handle this case in the future.
+        segment_id = 411
         sequence_number = 0
         payload_offset = 0
 
-        payload_length = self.get_payload_length()
+        remaining = len(self.payload)
+
+        # Every segment after Z=1 must repeat the LBox + TBox (first 8 bytes of the JUMBF box)
+        # immediately after CI + EN + Z. This frees 8 bytes from the max chunk size for those segments.
+        lbox_tbox = self.payload[0:8]
 
         app11_segments = []
 
-        while payload_length > 0:
-            sequence_number += 1
-            chunk_length = JPG_SEGMENT_MAX_PAYLOAD_LENGTH
-            if payload_length < JPG_SEGMENT_MAX_PAYLOAD_LENGTH:
-                chunk_length = payload_length
+        while remaining > 0:
+            sequence_number += 1  # The Z starts with 1.
+
+            if sequence_number == 1:
+                max_chunk = JPG_SEGMENT_MAX_PAYLOAD_LENGTH
+                chunk = self.payload[payload_offset : payload_offset + max_chunk]
+                segment_payload = chunk
+            else:
+                # Leave 8 bytes for the mandatory LBox + TBox prefix.
+                max_chunk = JPG_SEGMENT_MAX_PAYLOAD_LENGTH - 8
+                chunk = self.payload[payload_offset : payload_offset + max_chunk]
+                segment_payload = lbox_tbox + chunk
+
+            chunk_length = len(chunk)
 
             app11_segments.append(
                 JpgSegmentApp11(
                     segment_id=segment_id,
                     sequence_number=sequence_number,
-                    payload_length=chunk_length,
-                    payload=self.payload[payload_offset:],
+                    payload_length=len(segment_payload),
+                    payload=segment_payload,
                 )
             )
-            payload_length -= chunk_length
+
+            remaining -= chunk_length
             payload_offset += chunk_length
 
         serialized_storage_data = b""
@@ -116,8 +138,6 @@ def create_and_serialize_app11_storage(
     serialized_manifest_store = manifest_store.serialize()
 
     app11_storage = JpgSegmentApp11Storage(
-        app11_segment_box_length=manifest_store.get_length(),
-        app11_segment_box_type=manifest_store.get_type(),
         payload=serialized_manifest_store,
     )
 

c2pie/c2pa_parsing/manifest_extractor.py

@@ -38,7 +38,13 @@ def extract_manifest_store_bytes_from_jpeg(jpeg_bytes: bytes) -> bytes | None:
             z = int.from_bytes(payload[4:8], "big")
             if en not in first_offset:
                 first_offset[en] = i
-            streams[en].append((z, payload[8:]))
+
+            # Z = 1: payload[8:] includes LBox+TBox as the start of the JUMBF box.
+            # Z > 1: payload[8:16] is the repeated LBox+TBox prefix.
+            # We should skip it to get the continuation bytes only.
+            chunk = payload[8:] if z == 1 else payload[16:]
+
+            streams[en].append((z, chunk))
 
         i += 2 + seg_len
 

c2pie/interface.py

@@ -43,9 +43,6 @@ def c2pie_GenerateActionsAssertion(
     )
 
 
-# Currently not in use.
-# If APP11 exceeds the allowed size (65,535 bytes), an error will occur.
-# It is necessary to add logic to handle this case by splitting APP11.
 def c2pie_GenerateThumbnailAssertion(
     media_type: str,
     image_data: bytes,
@@ -56,9 +53,6 @@ def c2pie_GenerateThumbnailAssertion(
     )
 
 
-# Currently not in use.
-# If APP11 exceeds the allowed size (65,535 bytes), an error will occur.
-# It is necessary to add logic to handle this case by splitting APP11.
 def c2pie_GenerateIngredientThumbnailAssertion(
     media_type: str | None = None,
     image_data: bytes | None = None,

c2pie/main.py

@@ -91,14 +91,17 @@ def sign(arguments: argparse.Namespace) -> None:
     tsa_log_dir = arguments.tsa_log_dir or os.getenv("C2PIE_TSA_LOG_DIR")
     thumbnail_file_path = arguments.thumbnail_file
 
-    sign_file(
-        input_path=input_file_path,
-        output_path=output_file_path,
-        thumbnail_file_path=thumbnail_file_path,
-        tsa_url=tsa_url,
-        require_tsa=require_tsa,
-        tsa_log_dir=tsa_log_dir,
-    )
+    try:
+        sign_file(
+            input_path=input_file_path,
+            output_path=output_file_path,
+            thumbnail_file_path=thumbnail_file_path,
+            tsa_url=tsa_url,
+            require_tsa=require_tsa,
+            tsa_log_dir=tsa_log_dir,
+        )
+    except Exception as exception:
+        print(f"error: {exception}")
 
 
 def main() -> None:

c2pie/signing.py

@@ -16,7 +16,7 @@
     c2pie_GenerateThumbnailAssertion,
 )
 from c2pie.jumbf_boxes.box import Box
-from c2pie.utils.content_types import C2PA_ContentTypes, iana_media_types
+from c2pie.utils.content_types import C2PA_ContentTypes
 from c2pie.utils.generate_hashed_uri_map import generate_hashed_uri_map
 
 
@@ -31,9 +31,10 @@ def _get_content_type_by_filepath(file_path: Path) -> C2PA_ContentTypes:
     return file_content_type
 
 
-_DC_FORMAT_BY_CONTENT_TYPE: dict[str, str] = {
+_IANA_MEDIA_TYPES: dict[str, str] = {
     "jpg": "image/jpeg",
     "jpeg": "image/jpeg",
+    "png": "image/png",
     "pdf": "application/pdf",
 }
 
@@ -179,7 +180,7 @@ def sign_file(
         else:
             thumbnail_raw_bytes = _read_and_check_size_of_thumbnail_file(thumbnail_file_path)
 
-            thumbnail_media_type = iana_media_types[thumbnail_file_path.suffix]
+            thumbnail_media_type = _IANA_MEDIA_TYPES[thumbnail_file_path.suffix[1:]]
 
             thumbnail_assertion = c2pie_GenerateThumbnailAssertion(
                 thumbnail_media_type,
@@ -219,7 +220,7 @@ def sign_file(
 
     ingredient_assertion = c2pie_GenerateIngredientAssertion(
         title=input_path.name,
-        dc_format=_DC_FORMAT_BY_CONTENT_TYPE[file_type.name],
+        dc_format=_IANA_MEDIA_TYPES[file_type.name],
         ingredient_bytes=raw_bytes,
         active_manifest_urn=active_manifest_urn,
         active_manifest=active_manifest,

c2pie/utils/content_types.py

@@ -7,13 +7,6 @@ class C2PA_ContentTypes(enum.Enum):
     pdf = ".pdf"
 
 
-iana_media_types = {
-    ".jpeg": "image/jpeg",
-    ".jpg": "image/jpeg",
-    ".png": "image/png",
-    ".pdf": "application/pdf",
-}
-
 jumbf_content_types = {
     "embedded_file": b"\x40\xcb\x0c\x32\xbb\x8a\x48\x9d\xa7\x0b\x2a\xd6\xf4\x7f\x43\x69",
     "xml": b"\x78\x6d\x6c\x20\x00\x11\x00\x10\x80\x00\x00\xaa\x00\x38\x9b\x71",

tests/c2pa/manifest_store_test.py

@@ -124,3 +124,13 @@ def test_manifest_store_empty_list_equivalent_to_no_args():
     manifest_store = ManifestStore([])
     assert len(manifest_store.manifests) == 0
     assert len(manifest_store.content_boxes) == 0
+
+
+def test_serialize_raises_when_manifest_store_exceeds_lbox_limit():
+    import pytest
+
+    manifest_store = ManifestStore()
+    manifest_store.l_box = 0xFFFFFFFF + 1  # One byte over 4-byte max.
+
+    with pytest.raises(ValueError, match="Manifest Store is too large to serialize"):
+        manifest_store.serialize()

tests/c2pa_parsing/manifest_store_extractor_test.py

@@ -121,21 +121,59 @@ def test_binary_data_preserved(self):
 
 
 class TestFragmentedApp11:
+    def test_z1_segment_data_starts_at_byte_8_of_payload(self):
+        # For Z = 1 the full payload[8:] is the JUMBF data —
+        # LBox + TBox are included as-is, they are NOT a duplicated prefix here.
+        data = C2PA_MARK + b"lbox_tbox_and_more"
+        result = extract_manifest_store_bytes_from_jpeg(
+            make_jpeg(
+                make_app11(1, 1, data),
+            ),
+        )
+
+        assert result == data
+
+    def test_z_greater_than_1_skips_lbox_tbox_prefix(self):
+        # For Z > 1 the first 8 bytes after CI + EN + Z are the repeated LBox + TBox prefix
+        # and must be stripped; only bytes[16:] contain the continuation data.
+        part1 = C2PA_MARK + b"X" * 12  # 21 bytes total, first 8 = LBox+TBox
+        lbox_tbox = part1[:8]
+        continuation = b"real_data_here"
+
+        segs = make_jpeg(
+            make_app11(1, 1, part1),
+            make_app11(1, 2, lbox_tbox + continuation),
+        )
+
+        assert extract_manifest_store_bytes_from_jpeg(segs) == part1 + continuation
+
     def test_two_fragments_assembled_in_order(self):
         part1 = C2PA_MARK + b"first_"
         part2 = b"second"
-        segs = make_jpeg(make_app11(1, 1, part1), make_app11(1, 2, part2))
+        # Z > 1 segments carry a repeated LBox+TBox prefix (first 8 bytes of the box).
+        lbox_tbox = part1[:8]
+        segs = make_jpeg(make_app11(1, 1, part1), make_app11(1, 2, lbox_tbox + part2))
+
         assert extract_manifest_store_bytes_from_jpeg(segs) == part1 + part2
 
     def test_fragments_sorted_by_z_not_file_order(self):
         part1 = C2PA_MARK + b"AAA"
         part2 = b"BBB"
-        segs = make_jpeg(make_app11(1, 2, part2), make_app11(1, 1, part1))
+        lbox_tbox = part1[:8]
+        segs = make_jpeg(make_app11(1, 2, lbox_tbox + part2), make_app11(1, 1, part1))
+
         assert extract_manifest_store_bytes_from_jpeg(segs) == part1 + part2
 
     def test_many_fragments_assembled(self):
         chunks = [C2PA_MARK] + [bytes([i] * 100) for i in range(10)]
-        segs = make_jpeg(*[make_app11(1, z + 1, chunk) for z, chunk in enumerate(chunks)])
+        lbox_tbox = chunks[0][:8]
+
+        def make_fragment(z: int, chunk: bytes) -> bytes:
+            payload = chunk if z == 1 else lbox_tbox + chunk
+            return make_app11(1, z, payload)
+
+        segs = make_jpeg(*[make_fragment(z + 1, chunk) for z, chunk in enumerate(chunks)])
+
         assert extract_manifest_store_bytes_from_jpeg(segs) == b"".join(chunks)