[mpt] Add configurable root offsets chunk allocation

Chen-Yifan · Chen-Yifan · commit cd1df484c05d · 2025-11-21T19:42:48.000-05:00
Add --root-offsets-chunk-count flag to monad_mpt to make the number of
chunks allocated for root offsets configurable (default: 2, must be
power of 2). Each chunk holds approximately 16.5M history entries,
allowing operators to adjust history depth based on deployment
requirements. Previously, this was hardcoded to 2 chunks.
diff --git a/category/async/storage_pool.cpp b/category/async/storage_pool.cpp
@@ -80,6 +80,12 @@ size_t storage_pool::device_t::chunks() const
     return metadata_->chunks(size_of_file_);
 }
 
+size_t storage_pool::device_t::cnv_chunks() const
+{
+    MONAD_ASSERT(!is_zoned_device(), "zonefs support isn't implemented yet");
+    return metadata_->num_cnv_chunks;
+}
+
 std::pair<file_offset_t, file_offset_t> storage_pool::device_t::capacity() const
 {
     switch (type_) {
@@ -454,6 +460,7 @@ storage_pool::device_t storage_pool::make_device_(
             memcpy(metadata_footer->magic, "MND0", 4);
             metadata_footer->chunk_capacity =
                 static_cast<uint32_t>(chunk_capacity);
+            metadata_footer->num_cnv_chunks = flags.num_cnv_chunks;
             MONAD_ASSERT_PRINTF(
                 ::pwrite(
                     readwritefd,
@@ -504,21 +511,29 @@ void storage_pool::fill_chunks_(creation_flags const &flags)
         fnv1a_hash<uint32_t>::add(
             hashshouldbe, uint32_t(device.unique_hash_ >> 32));
     }
+    // Backward compatibility: databases created before `num_cnv_chunks` was
+    // added have this field set to 0. Treat 0 as the legacy default of 3
+    // chunks.
+    uint32_t const cnv_chunks_count =
+        devices_[0].metadata_->num_cnv_chunks == 0
+            ? 3
+            : devices_[0].metadata_->num_cnv_chunks;
     std::vector<size_t> chunks;
     size_t total = 0;
     chunks.reserve(devices_.size());
     for (auto const &device : devices_) {
         if (device.is_file() || device.is_block_device()) {
             auto const devicechunks = device.chunks();
             MONAD_ASSERT_PRINTF(
-                devicechunks >= 4,
-                "Device %s has %zu chunks the minimum allowed is four.",
+                devicechunks >= cnv_chunks_count + 1,
+                "Device %s has %zu chunks the minimum allowed is %u.",
                 device.current_path().c_str(),
-                devicechunks);
+                devicechunks,
+                cnv_chunks_count + 1);
             MONAD_ASSERT(devicechunks <= std::numeric_limits<uint32_t>::max());
-            // Take off three for the cnv chunks
-            chunks.push_back(devicechunks - 3);
-            total += devicechunks - 3;
+            // Take off cnv_chunks_count for the cnv chunks
+            chunks.push_back(devicechunks - cnv_chunks_count);
+            total += devicechunks - cnv_chunks_count;
             fnv1a_hash<uint32_t>::add(
                 hashshouldbe, static_cast<uint32_t>(devicechunks));
             fnv1a_hash<uint32_t>::add(
@@ -560,22 +575,17 @@ void storage_pool::fill_chunks_(creation_flags const &flags)
     auto const zone_id = [this](int const chunk_type) {
         return static_cast<uint32_t>(chunks_[chunk_type].size());
     };
-    // First three blocks of each device goes to conventional, remainder go to
-    // sequential
-    chunks_[cnv].reserve(devices_.size() * 3);
+    // First cnv_chunks_count blocks of each device goes to conventional,
+    // remainder go to sequential
+    chunks_[cnv].reserve(devices_.size() * cnv_chunks_count);
     chunks_[seq].reserve(total);
     if (flags.interleave_chunks_evenly) {
-        for (auto &device : devices_) {
-            chunks_[cnv].emplace_back(
-                activate_chunk(storage_pool::cnv, device, 0, zone_id(cnv)));
-        }
-        for (auto &device : devices_) {
-            chunks_[cnv].emplace_back(
-                activate_chunk(storage_pool::cnv, device, 1, zone_id(cnv)));
-        }
-        for (auto &device : devices_) {
-            chunks_[cnv].emplace_back(
-                activate_chunk(storage_pool::cnv, device, 2, zone_id(cnv)));
+        for (uint32_t chunk_idx = 0; chunk_idx < cnv_chunks_count;
+             ++chunk_idx) {
+            for (auto &device : devices_) {
+                chunks_[cnv].emplace_back(activate_chunk(
+                    storage_pool::cnv, device, chunk_idx, zone_id(cnv)));
+            }
         }
         // We now need to evenly spread the sequential chunks such that if
         // device A has 20, device B has 10 and device C has 5, the interleaving
@@ -585,7 +595,7 @@ void storage_pool::fill_chunks_(creation_flags const &flags)
         for (size_t n = 0; n < chunks.size(); n++) {
             chunkratios[n] = double(total) / static_cast<double>(chunks[n]);
             chunkcounts[n] = chunkratios[n];
-            chunks[n] = 3;
+            chunks[n] = cnv_chunks_count;
         }
         while (chunks_[seq].size() < chunks_[seq].capacity()) {
             for (size_t n = 0; n < chunks.size(); n++) {
@@ -612,19 +622,18 @@ void storage_pool::fill_chunks_(creation_flags const &flags)
     }
     else {
         for (auto &device : devices_) {
-            chunks_[cnv].emplace_back(
-                activate_chunk(cnv, device, 0, zone_id(cnv)));
-            chunks_[cnv].emplace_back(
-                activate_chunk(cnv, device, 1, zone_id(cnv)));
-            chunks_[cnv].emplace_back(
-                activate_chunk(cnv, device, 2, zone_id(cnv)));
+            for (uint32_t chunk_idx = 0; chunk_idx < cnv_chunks_count;
+                 ++chunk_idx) {
+                chunks_[cnv].emplace_back(
+                    activate_chunk(cnv, device, chunk_idx, zone_id(cnv)));
+            }
         }
         for (size_t deviceidx = 0; deviceidx < chunks.size(); deviceidx++) {
             for (size_t n = 0; n < chunks[deviceidx]; n++) {
                 chunks_[seq].emplace_back(activate_chunk(
                     seq,
                     devices_[deviceidx],
-                    static_cast<uint32_t>(3 + n),
+                    static_cast<uint32_t>(cnv_chunks_count + n),
                     zone_id(seq)));
             }
         }
diff --git a/category/async/storage_pool.hpp b/category/async/storage_pool.hpp
@@ -95,7 +95,8 @@ class storage_pool
         {
             // Preceding this is an array of uint32_t of chunk bytes used
 
-            uint32_t spare_[13]; // set aside for flags later
+            uint32_t spare_[12]; // set aside for flags later
+            uint32_t num_cnv_chunks; // number of cnv chunks per device
             uint32_t config_hash; // hash of this configuration
             uint32_t chunk_capacity;
             uint8_t magic[4]; // "MND0" for v1 of this metadata
@@ -178,6 +179,9 @@ class storage_pool
 
         //! Returns the number of chunks on this device
         size_t chunks() const;
+
+        //! Returns the number of cnv chunks on this device
+        size_t cnv_chunks() const;
         //! Returns the capacity of the device, and how much of that is
         //! currently filled with data, in that order.
         std::pair<file_offset_t, file_offset_t> capacity() const;
@@ -321,12 +325,16 @@ class storage_pool
         //! happily use any partition you feed it, including the system drive.
         uint32_t disable_mismatching_storage_pool_check : 1;
 
+        //! Number of conventional chunks to allocate per device. Default is 3.
+        uint32_t num_cnv_chunks;
+
         constexpr creation_flags()
             : chunk_capacity(28)
             , interleave_chunks_evenly(false)
             , open_read_only(false)
             , open_read_only_allow_dirty(false)
             , disable_mismatching_storage_pool_check(false)
+            , num_cnv_chunks(3)
         {
         }
     };
diff --git a/category/mpt/cli_tool_impl.cpp b/category/mpt/cli_tool_impl.cpp
@@ -393,6 +393,7 @@ struct impl_t
     std::ostream &cerr;
     MONAD_ASYNC_NAMESPACE::storage_pool::creation_flags flags;
     uint8_t chunk_capacity = flags.chunk_capacity;
+    uint32_t root_offsets_chunk_count = 2;
     bool allow_dirty = false;
     bool no_prompt = false;
     bool create_database = false;
@@ -1445,6 +1446,12 @@ opened.
                 "set chunk capacity during database creation (default is 28, "
                 "1<<28 "
                 "= 256Mb, max is 31).");
+            cli.add_option(
+                "--root-offsets-chunk-count",
+                impl.root_offsets_chunk_count,
+                "Number of chunks to allocate for storing root offsets. "
+                "Must be a power of 2. Default is 2. "
+                "Each chunk holds approximately 16.5M history entries.");
             cli.add_flag(
                 "--chunk-increasing",
                 impl.create_chunk_increasing,
@@ -1478,6 +1485,7 @@ opened.
             impl.flags.open_read_only = true;
             impl.flags.open_read_only_allow_dirty =
                 impl.allow_dirty || !impl.archive_database.empty();
+            impl.flags.num_cnv_chunks = impl.root_offsets_chunk_count + 1;
             if (!impl.restore_database.empty()) {
                 if (!impl.archive_database.empty()) {
                     impl.cli_ask_question(
diff --git a/category/mpt/db.cpp b/category/mpt/db.cpp
@@ -122,10 +122,12 @@ AsyncIOContext::AsyncIOContext(ReadOnlyOnDiskDbConfig const &options)
 
 AsyncIOContext::AsyncIOContext(OnDiskDbConfig const &options)
     : pool{[&] -> async::storage_pool {
+        async::storage_pool::creation_flags pool_options;
+        pool_options.num_cnv_chunks = options.root_offsets_chunk_count + 1;
         auto len = options.file_size_db * 1024 * 1024 * 1024 + 24576;
         if (options.dbname_paths.empty()) {
             return async::storage_pool{
-                async::use_anonymous_sized_inode_tag{}, len};
+                async::use_anonymous_sized_inode_tag{}, len, pool_options};
         }
         // initialize db file on disk
         for (auto const &dbname_path : options.dbname_paths) {
@@ -145,7 +147,8 @@ AsyncIOContext::AsyncIOContext(OnDiskDbConfig const &options)
         return async::storage_pool{
             options.dbname_paths,
             options.append ? async::storage_pool::mode::open_existing
-                           : async::storage_pool::mode::truncate};
+                           : async::storage_pool::mode::truncate,
+            pool_options};
     }()}
     , read_ring{{options.uring_entries, options.sq_thread_cpu}}
     , write_ring{io::RingConfig{options.wr_buffers}}
diff --git a/category/mpt/ondisk_db_config.hpp b/category/mpt/ondisk_db_config.hpp
@@ -43,6 +43,10 @@ struct OnDiskDbConfig
     // fixed history length if contains value, otherwise rely on db to adjust
     // history length upon disk usage
     std::optional<uint64_t> fixed_history_length{std::nullopt};
+    // Number of chunks to allocate for root offsets when initializing the disk.
+    // Each chunk can hold 1 << 24 = 16777216 historical entries.
+    // This field must be power of 2.
+    uint32_t root_offsets_chunk_count{2};
 };
 
 struct ReadOnlyOnDiskDbConfig
diff --git a/category/mpt/test/update_aux_test.cpp b/category/mpt/test/update_aux_test.cpp
@@ -30,6 +30,8 @@
 #include <category/mpt/trie.hpp>
 #include <category/mpt/util.hpp>
 
+#include <unistd.h>
+
 using namespace std::chrono_literals;
 
 namespace
@@ -197,3 +199,57 @@ TEST(update_aux_test, root_offsets_fast_slow)
             "Detected corruption");
     }
 }
+
+TEST(update_aux_test, configurable_root_offset_chunks)
+{
+    std::filesystem::path const filename{
+        MONAD_ASYNC_NAMESPACE::working_temporary_directory() /
+        "monad_update_aux_test_XXXXXX"};
+    int const fd = ::mkstemp((char *)filename.native().data());
+    MONAD_ASSERT(fd != -1);
+    MONAD_ASSERT(-1 != ::ftruncate(fd, 8UL << 30)); // 8GB
+
+    monad::io::Ring ring1;
+    monad::io::Ring ring2;
+    monad::io::Buffers testbuf =
+        monad::io::make_buffers_for_segregated_read_write(
+            ring1,
+            ring2,
+            2,
+            4,
+            monad::async::AsyncIO::MONAD_IO_BUFFERS_READ_SIZE,
+            monad::async::AsyncIO::MONAD_IO_BUFFERS_WRITE_SIZE);
+    monad::async::storage_pool::creation_flags flags;
+    flags.num_cnv_chunks = 5;
+    {
+        // Create storage pool with 5 conventional chunks
+        monad::async::storage_pool pool(
+            std::span{&filename, 1},
+            monad::async::storage_pool::mode::truncate,
+            flags);
+        EXPECT_EQ(pool.chunks(monad::async::storage_pool::cnv), 5);
+
+        monad::async::AsyncIO testio(pool, testbuf);
+        monad::mpt::UpdateAux<> aux(testio);
+
+        // Verify that exactly 4 chunks were allocated to hold two copies of
+        // root offsets, since chunk 0 is used for metadata
+        EXPECT_TRUE(aux.db_metadata()->using_chunks_for_root_offsets);
+        EXPECT_EQ(aux.db_metadata()->root_offsets.storage_.cnv_chunks_len, 4);
+        EXPECT_EQ(aux.root_offsets().capacity(), 2ULL << 25);
+    }
+    {
+        // reopen storage_pool
+        monad::async::storage_pool pool(
+            std::span{&filename, 1},
+            monad::async::storage_pool::mode::open_existing,
+            flags);
+        EXPECT_EQ(pool.chunks(monad::async::storage_pool::cnv), 5);
+        monad::async::AsyncIO testio(pool, testbuf);
+        monad::mpt::UpdateAux<> aux(testio);
+        EXPECT_TRUE(aux.db_metadata()->using_chunks_for_root_offsets);
+        EXPECT_EQ(aux.db_metadata()->root_offsets.storage_.cnv_chunks_len, 4);
+        EXPECT_EQ(aux.root_offsets().capacity(), 2ULL << 25);
+    }
+    remove(filename);
+}
diff --git a/category/mpt/update_aux.cpp b/category/mpt/update_aux.cpp
@@ -787,6 +787,11 @@ void UpdateAuxImpl::set_io(
                 start_lifetime_as<chunk_offset_t>(
                     (chunk_offset_t *)reservation[1]),
                 db_version_history_storage_bytes / sizeof(chunk_offset_t)};
+            LOG_INFO(
+                "Database root offsets ring buffer is configured with {} "
+                "chunks, can hold up to {} historical entries.",
+                db_metadata()->root_offsets.storage_.cnv_chunks_len,
+                root_offsets().capacity());
         }
     };
     if (0 != memcmp(
@@ -830,10 +835,15 @@ void UpdateAuxImpl::set_io(
             db_metadata_[0].main->using_chunks_for_root_offsets = true;
             db_metadata_[0].main->history_length =
                 chunk.capacity() / 2 / sizeof(chunk_offset_t);
-            // Gobble up all remaining cnv chunks
-            for (uint32_t n = 2;
-                 n < io->storage_pool().chunks(storage_pool::cnv);
-                 n++) {
+            // Allocate cnv chunks of the first device - 1 for root offsets
+            auto const root_offsets_chunk_count =
+                io->storage_pool().devices()[0].cnv_chunks() - 1;
+            MONAD_ASSERT(
+                root_offsets_chunk_count > 0 &&
+                    (root_offsets_chunk_count &
+                     (root_offsets_chunk_count - 1)) == 0,
+                "Number of cnv chunks for root offsets must be a power of two");
+            for (uint32_t n = 2; n <= root_offsets_chunk_count; n++) {
                 auto &chunk = io->storage_pool().chunk(storage_pool::cnv, n);
                 auto fdw = chunk.write_fd(chunk.capacity());
                 MONAD_ASSERT(
