From 38249ce9d58125136e7ed0d417c057a697526e2f Mon Sep 17 00:00:00 2001
From: Timm Ruppert <timm.ruppert@persival.de>
Date: Thu, 10 Oct 2024 16:26:26 +0200
Subject: [PATCH] MCAP spec proposal added

Signed-off-by: Timm Ruppert <timm.ruppert@persival.de>
---
 doc/architecture/formatting_script.adoc       | 29 -------
 ...le_binary_and_human_readable_formats.adoc} | 13 +++-
 doc/architecture/trace_file_example.adoc      | 78 -------------------
 doc/architecture/trace_file_mcap_format.adoc  | 39 ++++++++++
 ... => trace_file_overview_file_formats.adoc} | 17 ++--
 doc/open-simulation-interface_user_guide.adoc | 14 +---
 6 files changed, 62 insertions(+), 128 deletions(-)
 delete mode 100644 doc/architecture/formatting_script.adoc
 rename doc/architecture/{trace_file_naming.adoc => trace_file_binary_and_human_readable_formats.adoc} (73%)
 delete mode 100644 doc/architecture/trace_file_example.adoc
 create mode 100644 doc/architecture/trace_file_mcap_format.adoc
 rename doc/architecture/{trace_file_formats.adoc => trace_file_overview_file_formats.adoc} (52%)

diff --git a/doc/architecture/formatting_script.adoc b/doc/architecture/formatting_script.adoc
deleted file mode 100644
index d8beec595..000000000
--- a/doc/architecture/formatting_script.adoc
+++ /dev/null
@@ -1,29 +0,0 @@
-ifndef::include-only-once[]
-:root-path: ../
-include::{root-path}_config.adoc[]
-endif::[]
-= Trace-file formatting script
-
-The OSI repository contains a Python script for converting trace files from one format to another.
-The formatting script is stored in `open-simulation-interface/osi3trace/`.
-
-**osi2read.py**
-
-`osi2read.py` converts trace files to human-readable `.txth` trace files.
-This script takes the following parameters:
-
-`--data`, `-d`::
-String containing the path to the file with serialized data.
-
-`--type`, `-t`::
-Optional string describing the message type used to serialize data.
-`'SensorView'`, `'GroundTruth'`, or `'SensorData'` are permitted values.
-The default value is `'SensorView'`.
-
-`--output`, `-o`::
-Optional string containing the name of the output file.
-The default value is `None`, in which case the output file name is set to the name of the input file, with the file extension being replaced by `.txth`.
-
-**Related topics**
-
-* <<top-osi_trace_file_formats>>
\ No newline at end of file
diff --git a/doc/architecture/trace_file_naming.adoc b/doc/architecture/trace_file_binary_and_human_readable_formats.adoc
similarity index 73%
rename from doc/architecture/trace_file_naming.adoc
rename to doc/architecture/trace_file_binary_and_human_readable_formats.adoc
index 1c73ed90c..32135ca8c 100644
--- a/doc/architecture/trace_file_naming.adoc
+++ b/doc/architecture/trace_file_binary_and_human_readable_formats.adoc
@@ -2,11 +2,16 @@ ifndef::include-only-once[]
 :root-path: ../
 include::{root-path}_config.adoc[]
 endif::[]
-= OSI trace file naming conventions
+= Native binary and Human-readable Formats
 
-**Name format**
+== Binary .osi Format
+Messages are separated by a four-byte, little-endian, unsigned integer specifying the length of each message.
 
-The names of OSI trace files should have the following format:
+== Human-readable .txth Format
+Messages are stored as plain text, separated by newlines.
+
+== Naming Convention
+Binary .osi and human-readable .txth files should follow this naming convention:
 
 ----
 <timestamp>_<type>_<osi-version>_<protobuf-version>_<number-of-frames>_<custom-trace-name>.osi
@@ -76,3 +81,5 @@ The recommended file name is:
 ----
 20210818T150542Z_sv_312_300_1523_highway.osi
 ----
+
+NOTE: This naming convention does not apply to .mcap files.
\ No newline at end of file
diff --git a/doc/architecture/trace_file_example.adoc b/doc/architecture/trace_file_example.adoc
deleted file mode 100644
index 9de8dd43b..000000000
--- a/doc/architecture/trace_file_example.adoc
+++ /dev/null
@@ -1,78 +0,0 @@
-ifndef::include-only-once[]
-:root-path: ../
-include::{root-path}_config.adoc[]
-endif::[]
-[#top-osi_trace_file_example]
-= OSI trace file example
-
-[source, python]
-----
-# generate_osi_messages.py
-from osi3.osi_sensorview_pb2 import SensorView
-import struct
-
-NANO_INCREMENT = 10000000
-MOVING_OBJECT_LENGTH = 5
-MOVING_OBJECT_WIDTH = 2
-MOVING_OBJECT_HEIGHT = 1
-
-def main():
-    """Initialize SensorView"""
-    f = open("sv_330_361_1000_movingobject.osi", "ab")
-    sensorview = SensorView()
-
-    sv_ground_truth = sensorview.global_ground_truth
-    sv_ground_truth.version.version_major = 3
-    sv_ground_truth.version.version_minor = 5
-    sv_ground_truth.version.version_patch = 0
-
-    sv_ground_truth.timestamp.seconds = 0
-    sv_ground_truth.timestamp.nanos = 0
-
-    moving_object = sv_ground_truth.moving_object.add()
-    moving_object.id.value = 42
-
-    # Generate 1000 OSI messages for a duration of 10 seconds
-    for i in range(1000):
-
-        # Increment the time
-        if sv_ground_truth.timestamp.nanos > 1000000000:
-            sv_ground_truth.timestamp.seconds += 1
-            sv_ground_truth.timestamp.nanos = 0
-        sv_ground_truth.timestamp.nanos += NANO_INCREMENT
-
-        moving_object.vehicle_classification.type = 2
-
-        moving_object.base.dimension.length = MOVING_OBJECT_LENGTH
-        moving_object.base.dimension.width = MOVING_OBJECT_WIDTH
-        moving_object.base.dimension.height = MOVING_OBJECT_HEIGHT
-
-        moving_object.base.position.x += 0.5
-        moving_object.base.position.y = 0.0
-        moving_object.base.position.z = 0.0
-
-        moving_object.base.orientation.roll = 0.0
-        moving_object.base.orientation.pitch = 0.0
-        moving_object.base.orientation.yaw = 0.0
-
-        """Serialize"""
-        bytes_buffer = sensorview.SerializeToString()
-        f.write(struct.pack("<L", len(bytes_buffer)))
-        f.write(bytes_buffer)
-
-    f.close()
-
-if __name__ == "__main__":
-    main()
-----
-
-To run the script execute the following command in the terminal:
-
-[source, bash]
-----
-python3 generate_osi_messages.py
-----
-
-This will output an osi file (`sv_330_361_1000_movingobject.osi`) which can be visualized and played back by the https://github.com/OpenSimulationInterface/osi-visualizer[osi-visualizer^].
-
-See Google's documentation for more tutorials on how to use protocol buffers with https://developers.google.com/protocol-buffers/docs/pythontutorial[Python^] or https://developers.google.com/protocol-buffers/docs/cpptutorial[C++^].
\ No newline at end of file
diff --git a/doc/architecture/trace_file_mcap_format.adoc b/doc/architecture/trace_file_mcap_format.adoc
new file mode 100644
index 000000000..036126d36
--- /dev/null
+++ b/doc/architecture/trace_file_mcap_format.adoc
@@ -0,0 +1,39 @@
+= MCAP Format
+
+== General Requirements
+- Must comply with the https://mcap.dev/spec[MCAP format specification] version `0x30`
+- Must allow other non-OSI data to be present in the MCAP file
+- Message records must be written into `chunk records` for indexed files
+
+== Schema
+- `name` field: Full message type name, including package (e.g., `osi3.SensorData`)
+- `encoding` field: Must be `protobuf`
+- `data` field: String-encoded `google::protobuf::FileDescriptorSet` for the OSI top-level message
+
+== Channel
+- `message_encoding` field: Must be "protobuf"
+- `metadata` field: Should include a brief description of data origin and purpose
+
+== Message
+- `publish_time` field: 
+  - Must reflect the timestamp of the stored OSI top-level message if it contains a timestamp field
+  - Should represent a plausible value to express message order in relation to timestamps of other top-level messages if the message doesn't contain a timestamp (e.g. SensorViewConfiguration)
+  - Must be in nanoseconds
+- `log_time` field: Must reflect the time when the message was enqueued for MCAP file addition
+  - Must reflect the time when the top-level OSI message was enqueued for addition to the MCAP file
+  - Zero time doesn't need to coincide with the Unix epoch, but it is recommended
+  - Must be in nanoseconds
+
+== Metadata
+- Must include metadata with the name `versions` containing at least the following key-value pairs:
+  * `osi`: SemVer version of OSI specification
+  * `protobuf`: SemVer version of protobuf library
+- Must include metadata with the name `creation_date` containing at least the following key-value pairs:
+  * `timestamp`: ISO 8601 formatted creation time
+- Must include metadata with the name `description` containing at least the following key-value pairs:
+  * `text`: Scenario description
+- Additional custom metadata may be added
+
+== Compression
+- OSI-compliant tooling must support compression types: `none`, `lz4`, and `zstd`
+- Recommended compression in use should be either `lz4` or `zstd`
\ No newline at end of file
diff --git a/doc/architecture/trace_file_formats.adoc b/doc/architecture/trace_file_overview_file_formats.adoc
similarity index 52%
rename from doc/architecture/trace_file_formats.adoc
rename to doc/architecture/trace_file_overview_file_formats.adoc
index 60f2d2a4b..317ad15b2 100644
--- a/doc/architecture/trace_file_formats.adoc
+++ b/doc/architecture/trace_file_overview_file_formats.adoc
@@ -3,22 +3,25 @@ ifndef::include-only-once[]
 include::{root-path}_config.adoc[]
 endif::[]
 [#top-osi_trace_file_formats]
-= OSI trace file formats
+= Overview file formats
 
-There are two formats for storing multiple serialized OSI messages in one trace file.
+There are three formats for storing OSI messages in trace files:
 
 *.osi::
-Binary trace file.
-Messages are separated by a length specification before each message.
-The length is represented by a four-byte, little-endian, unsigned integer.
-The length does not include the integer itself.
+Native binary trace file.
 
 *.txth::
 Human-readable plain-text trace file.
-Messages are separated by newlines.
+
+*.mcap::
+Binary trace file supporting more advanced features like indexed data, additional metadata and more.
 
 NOTE: Previous releases of OSI also supported a so-called plain-text trace file format, with file extension `.txt`.
 This legacy format did not contain plain-text, but rather binary protobuf messages separated by a special separator.
 For obvious reasons the format was deprecated and fully replaced with the `.osi` binary file format.
 This release no longer contains any support for the legacy `.txt` file format.
 These files may be used for manual checks.
+
+
+TIP: For efficient handling of trace files, you can utilize the specification-compliant tools and libraries provided in the companion https://github.com/OpenSimulationInterface/osi-utilities[osi-utilities] repository. For example, convenient writer and reader classes are provided handling OSI messages for the different file formats.
+
diff --git a/doc/open-simulation-interface_user_guide.adoc b/doc/open-simulation-interface_user_guide.adoc
index 69296d267..8f5719ee5 100644
--- a/doc/open-simulation-interface_user_guide.adoc
+++ b/doc/open-simulation-interface_user_guide.adoc
@@ -57,19 +57,11 @@ include::./architecture/packaging_layer.adoc[leveloffset=+3]
 
 === OSI trace files
 
-include::./architecture/trace_file_formats.adoc[leveloffset=+3]
+include::./architecture/trace_file_overview_file_formats.adoc[leveloffset=+3]
 
-include::./architecture/trace_file_naming.adoc[leveloffset=+3]
+include::./architecture/trace_file_binary_and_human_readable_formats.adoc[leveloffset=+3]
 
-// === Files and scripts
-
-// include::./architecture/proto-files.adoc[leveloffset=+3]
-
-// include::./architecture/test_scripts.adoc[leveloffset=+3]
-
-include::./architecture/trace_file_example.adoc[leveloffset=+3]
-
-include::./architecture/formatting_script.adoc[leveloffset=+3]
+include::./architecture/trace_file_mcap_format.adoc[leveloffset=+3]
 
 
 // Setting up OSI