[mpt] Add configurable root offsets chunk allocation

Chen-Yifan · Chen-Yifan · commit 0c8e2941c6d5 · 2025-11-24T15:03:19.000-05:00
Add --root-offsets-chunk-count flag to monad_mpt to make the number of
chunks allocated for root offsets configurable (default: 16, must be
power of 2). Each chunk holds approximately 16.5M history entries,
allowing operators to adjust history depth based on deployment
requirements. Previously, this was hardcoded to 2 chunks.
diff --git a/category/async/storage_pool.cpp b/category/async/storage_pool.cpp
@@ -41,6 +41,8 @@
 
 #include <stdlib.h>
 
+#include <quill/Quill.h>
+
 #include <asm-generic/ioctl.h>
 #include <fcntl.h>
 #include <linux/falloc.h>
@@ -80,6 +82,12 @@ size_t storage_pool::device_t::chunks() const
     return metadata_->chunks(size_of_file_);
 }
 
+size_t storage_pool::device_t::cnv_chunks() const
+{
+    MONAD_ASSERT(!is_zoned_device(), "zonefs support isn't implemented yet");
+    return metadata_->num_cnv_chunks;
+}
+
 std::pair<file_offset_t, file_offset_t> storage_pool::device_t::capacity() const
 {
     switch (type_) {
@@ -454,6 +462,7 @@ storage_pool::device_t storage_pool::make_device_(
             memcpy(metadata_footer->magic, "MND0", 4);
             metadata_footer->chunk_capacity =
                 static_cast<uint32_t>(chunk_capacity);
+            metadata_footer->num_cnv_chunks = flags.num_cnv_chunks;
             MONAD_ASSERT_PRINTF(
                 ::pwrite(
                     readwritefd,
@@ -465,6 +474,17 @@ storage_pool::device_t storage_pool::make_device_(
         }
         total_size =
             metadata_footer->total_size(static_cast<size_t>(stat.st_size));
+        if (flags.num_cnv_chunks > metadata_footer->num_cnv_chunks) {
+            LOG_WARNING(
+                "Flag-specified num_cnv_chunks ({}) is greater than the value "
+                "stored in metadata ({}). This setting will be ignored. "
+                "Existing databases cannot be reconfigured to use more chunks, "
+                "create a new database if you need a higher num_cnv_chunks.",
+                flags.num_cnv_chunks,
+                metadata_footer->num_cnv_chunks == 0
+                    ? 3
+                    : metadata_footer->num_cnv_chunks);
+        }
     }
     size_t const offset = round_down_align<CPU_PAGE_BITS>(
         static_cast<size_t>(stat.st_size) - total_size);
@@ -504,21 +524,29 @@ void storage_pool::fill_chunks_(creation_flags const &flags)
         fnv1a_hash<uint32_t>::add(
             hashshouldbe, uint32_t(device.unique_hash_ >> 32));
     }
+    // Backward compatibility: databases created before `num_cnv_chunks` was
+    // added have this field set to 0. Treat 0 as the legacy default of 3
+    // chunks.
+    uint32_t const cnv_chunks_count =
+        devices_[0].metadata_->num_cnv_chunks == 0
+            ? 3
+            : devices_[0].metadata_->num_cnv_chunks;
     std::vector<size_t> chunks;
     size_t total = 0;
     chunks.reserve(devices_.size());
     for (auto const &device : devices_) {
         if (device.is_file() || device.is_block_device()) {
             auto const devicechunks = device.chunks();
             MONAD_ASSERT_PRINTF(
-                devicechunks >= 4,
-                "Device %s has %zu chunks the minimum allowed is four.",
+                devicechunks >= cnv_chunks_count + 1,
+                "Device %s has %zu chunks the minimum allowed is %u.",
                 device.current_path().c_str(),
-                devicechunks);
+                devicechunks,
+                cnv_chunks_count + 1);
             MONAD_ASSERT(devicechunks <= std::numeric_limits<uint32_t>::max());
-            // Take off three for the cnv chunks
-            chunks.push_back(devicechunks - 3);
-            total += devicechunks - 3;
+            // Take off cnv_chunks_count for the cnv chunks
+            chunks.push_back(devicechunks - cnv_chunks_count);
+            total += devicechunks - cnv_chunks_count;
             fnv1a_hash<uint32_t>::add(
                 hashshouldbe, static_cast<uint32_t>(devicechunks));
             fnv1a_hash<uint32_t>::add(
@@ -560,22 +588,17 @@ void storage_pool::fill_chunks_(creation_flags const &flags)
     auto const zone_id = [this](int const chunk_type) {
         return static_cast<uint32_t>(chunks_[chunk_type].size());
     };
-    // First three blocks of each device goes to conventional, remainder go to
-    // sequential
-    chunks_[cnv].reserve(devices_.size() * 3);
+    // First cnv_chunks_count blocks of each device goes to conventional,
+    // remainder go to sequential
+    chunks_[cnv].reserve(devices_.size() * cnv_chunks_count);
     chunks_[seq].reserve(total);
     if (flags.interleave_chunks_evenly) {
-        for (auto &device : devices_) {
-            chunks_[cnv].emplace_back(
-                activate_chunk(storage_pool::cnv, device, 0, zone_id(cnv)));
-        }
-        for (auto &device : devices_) {
-            chunks_[cnv].emplace_back(
-                activate_chunk(storage_pool::cnv, device, 1, zone_id(cnv)));
-        }
-        for (auto &device : devices_) {
-            chunks_[cnv].emplace_back(
-                activate_chunk(storage_pool::cnv, device, 2, zone_id(cnv)));
+        for (uint32_t chunk_idx = 0; chunk_idx < cnv_chunks_count;
+             ++chunk_idx) {
+            for (auto &device : devices_) {
+                chunks_[cnv].emplace_back(activate_chunk(
+                    storage_pool::cnv, device, chunk_idx, zone_id(cnv)));
+            }
         }
         // We now need to evenly spread the sequential chunks such that if
         // device A has 20, device B has 10 and device C has 5, the interleaving
@@ -585,7 +608,7 @@ void storage_pool::fill_chunks_(creation_flags const &flags)
         for (size_t n = 0; n < chunks.size(); n++) {
             chunkratios[n] = double(total) / static_cast<double>(chunks[n]);
             chunkcounts[n] = chunkratios[n];
-            chunks[n] = 3;
+            chunks[n] = cnv_chunks_count;
         }
         while (chunks_[seq].size() < chunks_[seq].capacity()) {
             for (size_t n = 0; n < chunks.size(); n++) {
@@ -612,19 +635,18 @@ void storage_pool::fill_chunks_(creation_flags const &flags)
     }
     else {
         for (auto &device : devices_) {
-            chunks_[cnv].emplace_back(
-                activate_chunk(cnv, device, 0, zone_id(cnv)));
-            chunks_[cnv].emplace_back(
-                activate_chunk(cnv, device, 1, zone_id(cnv)));
-            chunks_[cnv].emplace_back(
-                activate_chunk(cnv, device, 2, zone_id(cnv)));
+            for (uint32_t chunk_idx = 0; chunk_idx < cnv_chunks_count;
+                 ++chunk_idx) {
+                chunks_[cnv].emplace_back(
+                    activate_chunk(cnv, device, chunk_idx, zone_id(cnv)));
+            }
         }
         for (size_t deviceidx = 0; deviceidx < chunks.size(); deviceidx++) {
             for (size_t n = 0; n < chunks[deviceidx]; n++) {
                 chunks_[seq].emplace_back(activate_chunk(
                     seq,
                     devices_[deviceidx],
-                    static_cast<uint32_t>(3 + n),
+                    static_cast<uint32_t>(cnv_chunks_count + n),
                     zone_id(seq)));
             }
         }
diff --git a/category/async/storage_pool.hpp b/category/async/storage_pool.hpp
@@ -95,7 +95,8 @@ class storage_pool
         {
             // Preceding this is an array of uint32_t of chunk bytes used
 
-            uint32_t spare_[13]; // set aside for flags later
+            uint32_t spare_[12]; // set aside for flags later
+            uint32_t num_cnv_chunks; // number of cnv chunks per device
             uint32_t config_hash; // hash of this configuration
             uint32_t chunk_capacity;
             uint8_t magic[4]; // "MND0" for v1 of this metadata
@@ -178,6 +179,9 @@ class storage_pool
 
         //! Returns the number of chunks on this device
         size_t chunks() const;
+
+        //! Returns the number of cnv chunks on this device
+        size_t cnv_chunks() const;
         //! Returns the capacity of the device, and how much of that is
         //! currently filled with data, in that order.
         std::pair<file_offset_t, file_offset_t> capacity() const;
@@ -321,12 +325,16 @@ class storage_pool
         //! happily use any partition you feed it, including the system drive.
         uint32_t disable_mismatching_storage_pool_check : 1;
 
+        //! Number of conventional chunks to allocate per device. Default is 3.
+        uint32_t num_cnv_chunks;
+
         constexpr creation_flags()
             : chunk_capacity(28)
             , interleave_chunks_evenly(false)
             , open_read_only(false)
             , open_read_only_allow_dirty(false)
             , disable_mismatching_storage_pool_check(false)
+            , num_cnv_chunks(3)
         {
         }
     };
diff --git a/category/mpt/cli_tool_impl.cpp b/category/mpt/cli_tool_impl.cpp
@@ -393,6 +393,7 @@ struct impl_t
     std::ostream &cerr;
     MONAD_ASYNC_NAMESPACE::storage_pool::creation_flags flags;
     uint8_t chunk_capacity = flags.chunk_capacity;
+    uint32_t root_offsets_chunk_count = 16;
     bool allow_dirty = false;
     bool no_prompt = false;
     bool create_database = false;
@@ -1445,6 +1446,22 @@ opened.
                 "set chunk capacity during database creation (default is 28, "
                 "1<<28 "
                 "= 256Mb, max is 31).");
+            cli.add_option(
+                   "--root-offsets-chunk-count",
+                   impl.root_offsets_chunk_count,
+                   "Number of chunks to allocate for storing root offsets. "
+                   "Must be a positive number that is a power of 2. Default is "
+                   "16. Each chunk holds approx 16.5M history entries.")
+                ->check([](std::string const &s) {
+                    auto const v = std::stoll(s);
+                    if (v <= 0) {
+                        return "Value must be positive";
+                    }
+                    if ((v & (v - 1)) != 0) {
+                        return "Value must be a power of 2";
+                    }
+                    return "";
+                });
             cli.add_flag(
                 "--chunk-increasing",
                 impl.create_chunk_increasing,
@@ -1478,6 +1495,9 @@ opened.
             impl.flags.open_read_only = true;
             impl.flags.open_read_only_allow_dirty =
                 impl.allow_dirty || !impl.archive_database.empty();
+            impl.flags.num_cnv_chunks =
+                impl.root_offsets_chunk_count +
+                monad::mpt::UpdateAuxImpl::cnv_chunks_for_db_metadata;
             if (!impl.restore_database.empty()) {
                 if (!impl.archive_database.empty()) {
                     impl.cli_ask_question(
diff --git a/category/mpt/db.cpp b/category/mpt/db.cpp
@@ -122,10 +122,12 @@ AsyncIOContext::AsyncIOContext(ReadOnlyOnDiskDbConfig const &options)
 
 AsyncIOContext::AsyncIOContext(OnDiskDbConfig const &options)
     : pool{[&] -> async::storage_pool {
+        async::storage_pool::creation_flags pool_options;
+        pool_options.num_cnv_chunks = options.root_offsets_chunk_count + 1;
         auto len = options.file_size_db * 1024 * 1024 * 1024 + 24576;
         if (options.dbname_paths.empty()) {
             return async::storage_pool{
-                async::use_anonymous_sized_inode_tag{}, len};
+                async::use_anonymous_sized_inode_tag{}, len, pool_options};
         }
         // initialize db file on disk
         for (auto const &dbname_path : options.dbname_paths) {
@@ -145,7 +147,8 @@ AsyncIOContext::AsyncIOContext(OnDiskDbConfig const &options)
         return async::storage_pool{
             options.dbname_paths,
             options.append ? async::storage_pool::mode::open_existing
-                           : async::storage_pool::mode::truncate};
+                           : async::storage_pool::mode::truncate,
+            pool_options};
     }()}
     , read_ring{{options.uring_entries, options.sq_thread_cpu}}
     , write_ring{io::RingConfig{options.wr_buffers}}
diff --git a/category/mpt/ondisk_db_config.hpp b/category/mpt/ondisk_db_config.hpp
@@ -43,6 +43,10 @@ struct OnDiskDbConfig
     // fixed history length if contains value, otherwise rely on db to adjust
     // history length upon disk usage
     std::optional<uint64_t> fixed_history_length{std::nullopt};
+    // Number of chunks to allocate for root offsets when initializing the disk.
+    // Each chunk can hold 1 << 24 = 16777216 historical entries.
+    // This field must be power of 2.
+    uint32_t root_offsets_chunk_count{2};
 };
 
 struct ReadOnlyOnDiskDbConfig
diff --git a/category/mpt/test/cli_tool_test.cpp b/category/mpt/test/cli_tool_test.cpp
@@ -73,7 +73,7 @@ TEST(cli_tool, create)
     ::close(fd);
     auto untempfile =
         monad::make_scope_exit([&]() noexcept { unlink(temppath); });
-    if (-1 == truncate(temppath, 2ULL * 1024 * 1024 * 1024)) {
+    if (-1 == truncate(temppath, 6ULL * 1024 * 1024 * 1024)) {
         abort();
     }
     std::cout << "temp file being used: " << temppath << std::endl;
@@ -108,6 +108,8 @@ struct cli_tool_fixture
 {
     void run_test()
     {
+        constexpr unsigned default_num_cnv_chunks = 17;
+
         char temppath1[] = "cli_tool_test_XXXXXX";
         char dbpath2a[] = "cli_tool_test_XXXXXX";
         char dbpath2b[] = "cli_tool_test_XXXXXX";
@@ -152,15 +154,15 @@ struct cli_tool_fixture
         if (Config.interleave_multiple_sources) {
             if (-1 == truncate(
                           dbpath2a,
-                          (4 + Config.chunks_max / 2) *
+                          (default_num_cnv_chunks + Config.chunks_max / 2) *
                                   MONAD_ASYNC_NAMESPACE::AsyncIO::
                                       MONAD_IO_BUFFERS_WRITE_SIZE +
                               24576)) {
                 abort();
             }
             if (-1 == truncate(
                           dbpath2b,
-                          (4 + Config.chunks_max / 2) *
+                          (default_num_cnv_chunks + Config.chunks_max / 2) *
                                   MONAD_ASYNC_NAMESPACE::AsyncIO::
                                       MONAD_IO_BUFFERS_WRITE_SIZE +
                               24576)) {
@@ -170,12 +172,12 @@ struct cli_tool_fixture
             dbpath2.push_back(dbpath2b);
         }
         else {
-            if (-1 ==
-                truncate(
-                    dbpath2a,
-                    (3 + Config.chunks_max) * MONAD_ASYNC_NAMESPACE::AsyncIO::
-                                                  MONAD_IO_BUFFERS_WRITE_SIZE +
-                        24576)) {
+            if (-1 == truncate(
+                          dbpath2a,
+                          (default_num_cnv_chunks + Config.chunks_max) *
+                                  MONAD_ASYNC_NAMESPACE::AsyncIO::
+                                      MONAD_IO_BUFFERS_WRITE_SIZE +
+                              24576)) {
                 abort();
             }
             dbpath2.push_back(dbpath2a);
@@ -257,12 +259,12 @@ struct cli_tool_fixture
             if (-1 == fd) {
                 abort();
             }
-            if (-1 ==
-                ftruncate(
-                    fd,
-                    (3 + Config.chunks_max) * MONAD_ASYNC_NAMESPACE::AsyncIO::
-                                                  MONAD_IO_BUFFERS_WRITE_SIZE +
-                        24576)) {
+            if (-1 == ftruncate(
+                          fd,
+                          (default_num_cnv_chunks + Config.chunks_max) *
+                                  MONAD_ASYNC_NAMESPACE::AsyncIO::
+                                      MONAD_IO_BUFFERS_WRITE_SIZE +
+                              24576)) {
                 abort();
             }
             ::close(fd);
diff --git a/category/mpt/test/update_aux_test.cpp b/category/mpt/test/update_aux_test.cpp
diff --git a/category/mpt/trie.hpp b/category/mpt/trie.hpp
diff --git a/category/mpt/update_aux.cpp b/category/mpt/update_aux.cpp
