rosbag2_storage_mcap: add storage preset profiles (ros2#57)

* rosbag2_storage_mcap: add storage preset profiles Signed-off-by: James Smith <james@foxglove.dev> * mcap_vendor: update to mcap v0.5.0 Signed-off-by: James Smith <james@foxglove.dev> Signed-off-by: James Smith <james@foxglove.dev>
james-rms · Nov 17, 2022 · 3d24d28 · 3d24d28
1 parent 5cbd9f1
commit 3d24d28
Show file tree

Hide file tree

Showing 3 changed files with 105 additions and 9 deletions.
diff --git a/mcap_vendor/CMakeLists.txt b/mcap_vendor/CMakeLists.txt
@@ -21,7 +21,7 @@ macro(build_mcap_vendor)
   include(FetchContent)
   fetchcontent_declare(mcap
     GIT_REPOSITORY https://github.com/foxglove/mcap.git
-    GIT_TAG 24b679553674de7b04e87501abdfcde33756260f # releases/cpp/v0.2.0
+    GIT_TAG da2b85b9a6745a7022ff2736fb7d159c1c027087 # releases/cpp/v0.5.0
   )
   fetchcontent_makeavailable(mcap)
 

diff --git a/rosbag2_storage_mcap/README.md b/rosbag2_storage_mcap/README.md
@@ -36,7 +36,10 @@ To configure details of the MCAP writer for `ros2 bag record`, use the `--storag
 
 | Field | Type / Values | Description |
 | ----- | ------------- | ----------- |
-| noCRC | bool | Disable CRC calculations for Chunks, Attachments, and the Data and Summary sections. |
+| noChunkCRC | bool | Disable CRC calculation for Chunks. Ignored if `noChunking=true`. |
+| noAttachmentCRC | bool | Disable CRC calculation for Attachments. |
+| enableDataCRC | bool | Enables CRC calculation for the entire Data section. Useful when `noChunking=True`. |
+| noSummaryCRC | bool | Disable CRC calculation for the Summary section. |
 | noChunking | bool | Do write Chunks to the file, instead writing Schema, Channel, and Message records directly into the Data section. |
 | noMessageIndex | bool | Do not write Message Index records to the file. If `noSummary=true` and `noChunkIndex=false`, Chunk Index records will still be written to the Summary section, providing a coarse message index. |
 | noSummary | bool | Do not write Summary or Summary Offset sections to the file, placing the Footer record immediately after DataEnd. This can provide some speed boost to file writing and produce smaller files, at the expense of requiring a conversion process later if fast summarization or indexed access is desired. |
@@ -56,7 +59,7 @@ Example:
 
 ```
 # mcap_writer_options.yml
-noCRC: false
+noChunkCRC: false
 noChunking: false
 noMessageIndex: false
 noSummary: false
@@ -70,6 +73,68 @@ forceCompression: false
 $ ros2 bag record -s mcap -o my_bag --all --storage-config-file mcap_writer_options.yml
 ```
 
+### Storage Preset Profiles
+
+You can also use one of the preset profiles described below, for example:
+
+```
+$ ros2 bag record -s mcap -o my_bag --all --storage-preset-profile fastwrite
+```
+
+#### `fastwrite`
+
+Configures the MCAP writer for the highest possible write throughput and lowest resource utilization. This preset does not calculate CRCs for integrity checking, and does not write a message index. This preset profile is useful for resource-constrained robots.
+
+Equivalent to this storage configuration:
+```yaml
+noChunking: true
+noSummaryCRC: true
+```
+
+Using MCAPs written with `fastwrite` as a long-term storage format is not recommended. Some features will not work when reading MCAP files without a message index, such as reading messages from a subset of topics or seeking. When recording MCAPs on your robot with `fastwrite`, it is a good idea to post-process these files afterwards, to restore the message index and also save storage space:
+
+```bash
+# Using the MCAP CLI https://github.com/foxglove/mcap/tree/main/go/cli/mcap
+$ mcap compress fast.mcap -o compressed.mcap
+# Using `ros2 bag convert`
+$ cat << EOF > convert.yaml
+output_bags:
+  - uri: compressed
+    storage_id: mcap
+    storage_preset_profile: zstd_small
+EOF
+$ ros2 bag convert -i fast.mcap -o convert.yaml
+```
+
+Equivalent to this storage configuration:
+```yaml
+noChunking: true
+noSummaryCRC: true
+```
+
+#### `zstd_fast`
+
+Configures the MCAP writer to use chunk compression with [zstd](http://facebook.github.io/zstd/). Chunk compression yields file sizes comparable to bags compressed with file-level compression, but allows tools to efficiently read messages without decompressing the entire bag. This preset uses the lowest compression ratio and disables CRC calculation, to achieve high throughput while conserving disk space.
+
+Equivalent to this storage configuration:
+
+```yaml
+compression: "Zstd"
+compressionLevel: "Fastest"
+noChunkCRC: true
+```
+
+#### `zstd_small`
+
+Configures the MCAP writer to write 4MB chunks, compressed with zstd using its highest compression ratio. This produces very small bags, but can be resource-intensive to write. This preset also calculates chunk CRCs, which allow a reader to determine if a chunk is corrupted. This preset is useful when using `ros2 bag convert` as a post-processing step.
+
+Equivalent to this storage configuration:
+
+```yaml
+compression: "Zstd"
+compressionLevel: "Slowest"
+chunkSize: 4194304 # 4 * 1024 * 1024
+```
 
 ## Development
 

diff --git a/rosbag2_storage_mcap/src/mcap_storage.cpp b/rosbag2_storage_mcap/src/mcap_storage.cpp
@@ -113,7 +113,10 @@ template <>
 struct convert<McapWriterOptions> {
   // NOTE: when updating this struct, also update documentation in README.md
   static bool decode(const Node& node, McapWriterOptions& o) {
-    optional_assign<bool>(node, "noCRC", o.noCRC);
+    optional_assign<bool>(node, "noChunkCRC", o.noChunkCRC);
+    optional_assign<bool>(node, "noAttachmentCRC", o.noAttachmentCRC);
+    optional_assign<bool>(node, "enableDataCRC", o.enableDataCRC);
+    optional_assign<bool>(node, "noSummaryCRC", o.noSummaryCRC);
     optional_assign<bool>(node, "noChunking", o.noChunking);
     optional_assign<bool>(node, "noMessageIndex", o.noMessageIndex);
     optional_assign<bool>(node, "noSummary", o.noSummary);
@@ -201,7 +204,8 @@ class MCAPStorage : public rosbag2_storage::storage_interfaces::ReadWriteInterfa
   void remove_topic(const rosbag2_storage::TopicMetadata& topic) override;
 
 private:
-  void open_impl(const std::string& uri, rosbag2_storage::storage_interfaces::IOFlag io_flag,
+  void open_impl(const std::string& uri, const std::string& preset_profile,
+                 rosbag2_storage::storage_interfaces::IOFlag io_flag,
                  const std::string& storage_config_uri);
 
   void reset_iterator(rcutils_time_point_value_t start_time = 0);
@@ -254,16 +258,37 @@ MCAPStorage::~MCAPStorage() {
 #ifdef ROSBAG2_STORAGE_MCAP_HAS_STORAGE_OPTIONS
 void MCAPStorage::open(const rosbag2_storage::StorageOptions& storage_options,
                        rosbag2_storage::storage_interfaces::IOFlag io_flag) {
-  open_impl(storage_options.uri, io_flag, storage_options.storage_config_uri);
+  open_impl(storage_options.uri, storage_options.storage_preset_profile, io_flag,
+            storage_options.storage_config_uri);
 }
 #endif
 
 void MCAPStorage::open(const std::string& uri,
                        rosbag2_storage::storage_interfaces::IOFlag io_flag) {
-  open_impl(uri, io_flag, "");
+  open_impl(uri, "", io_flag, "");
 }
 
-void MCAPStorage::open_impl(const std::string& uri,
+static void SetOptionsForPreset(const std::string& preset_profile, McapWriterOptions& options) {
+  if (preset_profile == "fastwrite") {
+    options.noChunking = true;
+    options.noSummaryCRC = true;
+  } else if (preset_profile == "zstd_fast") {
+    options.compression = mcap::Compression::Zstd;
+    options.compressionLevel = mcap::CompressionLevel::Fastest;
+    options.noChunkCRC = true;
+  } else if (preset_profile == "zstd_small") {
+    options.compression = mcap::Compression::Zstd;
+    options.compressionLevel = mcap::CompressionLevel::Slowest;
+    options.chunkSize = 4 * 1024 * 1024;
+  } else {
+    throw std::runtime_error(
+      "unknown MCAP storage preset profile "
+      "(valid options are 'fastwrite', 'zstd_fast', 'zstd_small'): " +
+      preset_profile);
+  }
+}
+
+void MCAPStorage::open_impl(const std::string& uri, const std::string& preset_profile,
                             rosbag2_storage::storage_interfaces::IOFlag io_flag,
                             const std::string& storage_config_uri) {
   switch (io_flag) {
@@ -287,9 +312,15 @@ void MCAPStorage::open_impl(const std::string& uri,
 
       mcap_writer_ = std::make_unique<mcap::McapWriter>();
       McapWriterOptions options;
+      // Set options from preset profile first
+      if (!preset_profile.empty()) {
+        SetOptionsForPreset(preset_profile, options);
+      }
+      // If both preset profile and storage config are specified,
+      // options from the storage config are overlaid on the options from the preset profile.
       if (!storage_config_uri.empty()) {
         YAML::Node yaml_node = YAML::LoadFile(storage_config_uri);
-        options = yaml_node.as<McapWriterOptions>();
+        YAML::convert<McapWriterOptions>::decode(yaml_node, options);
       }
 
       auto status = mcap_writer_->open(relative_path_, options);