tool: Show the entries from wal log files (#6224)

close #5568

dbms/src/Storages/Page/Config.h

@@ -145,7 +145,7 @@ struct PageStorageConfig
             "PageStorageConfig V3 {{"
             "blob_file_limit_size: {}, blob_spacemap_type: {}, "
             "blob_cached_fd_size: {}, blob_heavy_gc_valid_rate: {:.3f}, blob_block_alignment_bytes: {}, "
-            "wal_roll_size: {},  wal_max_persisted_log_files: {}}}",
+            "wal_roll_size: {}, wal_max_persisted_log_files: {}}}",
             blob_file_limit_size.get(),
             blob_spacemap_type.get(),
             blob_cached_fd_size.get(),

dbms/src/Storages/Page/V3/PageDirectory.cpp

@@ -478,7 +478,7 @@ PageSize VersionedPageEntries::getEntriesByBlobIds(
     auto page_lock = acquireLock();
     if (type == EditRecordType::VAR_ENTRY)
     {
-        for (const auto & [versioned_type, entry_or_del] : entries)
+        for (const auto & [ver, entry_or_del] : entries)
         {
             if (!entry_or_del.isEntry())
             {
@@ -488,7 +488,7 @@ PageSize VersionedPageEntries::getEntriesByBlobIds(
             const auto & entry = entry_or_del.entry;
             if (blob_ids.count(entry.file_id) > 0)
             {
-                blob_versioned_entries[entry.file_id].emplace_back(page_id, versioned_type, entry);
+                blob_versioned_entries[entry.file_id].emplace_back(page_id, ver, entry);
                 total_entries_size += entry.size;
             }
         }
@@ -1286,7 +1286,7 @@ PageDirectory::getEntriesByBlobIds(const std::vector<BlobFileId> & blob_ids) con
         }
     }
 
-    LOG_INFO(log, "Get entries by Blob ids done. [total_page_size={}] [total_page_nums={}]", //
+    LOG_INFO(log, "Get entries by blob ids done. [total_page_size={}] [total_page_nums={}]", //
              total_page_size, //
              total_page_nums);
     return std::make_pair(std::move(blob_versioned_entries), total_page_size);

dbms/src/Storages/Page/V3/PageDirectoryFactory.cpp

@@ -121,6 +121,8 @@ void PageDirectoryFactory::loadEdit(const PageDirectoryPtr & dir, const PageEntr
         if (max_applied_ver < r.version)
             max_applied_ver = r.version;
 
+        if (dump_entries)
+            LOG_INFO(Logger::get(), PageEntriesEdit::toDebugString(r));
         applyRecord(dir, r);
     }
 }

dbms/src/Storages/Page/V3/PageDirectoryFactory.h

@@ -69,6 +69,10 @@ class PageDirectoryFactory
         const PageEntriesEdit::EditRecord & r);
 
     BlobStats * blob_stats = nullptr;
+
+    // For debug tool
+    friend class PageStorageControlV3;
+    bool dump_entries = false;
 };
 
 } // namespace PS::V3

dbms/src/Storages/Page/V3/tests/gtest_page_storage.cpp

@@ -55,6 +55,7 @@ class PageStorageTest : public DB::base::TiFlashStorageTestBasic
     void SetUp() override
     {
         TiFlashStorageTestBasic::SetUp();
+        log = Logger::get();
         auto path = getTemporaryPath();
         createIfNotExist(path);
         file_provider = DB::tests::TiFlashTestEnv::getContext().getFileProvider();
@@ -72,6 +73,11 @@ class PageStorageTest : public DB::base::TiFlashStorageTestBasic
         return storage;
     }
 
+    size_t getLogFileNum()
+    {
+        auto log_files = WALStoreReader::listAllFiles(delegator, log);
+        return log_files.size();
+    }
 
 protected:
     FileProviderPtr file_provider;
@@ -80,10 +86,7 @@ class PageStorageTest : public DB::base::TiFlashStorageTestBasic
     PageStorageConfig config;
     std::shared_ptr<PageStorageImpl> page_storage;
 
-    std::list<PageDirectorySnapshotPtr> snapshots_holder;
-    size_t fixed_test_buff_size = 1024;
-
-    size_t epoch_offset = 0;
+    LoggerPtr log;
 };
 
 TEST_F(PageStorageTest, WriteRead)
@@ -1646,13 +1649,8 @@ try
         page_storage->write(std::move(batch));
     }
 
-    auto get_log_file_num = [&]() {
-        auto log_files = WALStoreReader::listAllFiles(delegator, Logger::get());
-        return log_files.size();
-    };
-
     // write until there are more than one wal file
-    while (get_log_file_num() <= 1)
+    while (getLogFileNum() <= 1)
     {
         WriteBatch batch;
         PageId page_id1 = 130;
@@ -1727,13 +1725,8 @@ try
         page_storage->write(std::move(batch));
     }
 
-    auto get_log_file_num = [&]() {
-        auto log_files = WALStoreReader::listAllFiles(delegator, Logger::get());
-        return log_files.size();
-    };
-
     // write until there are more than one wal file
-    while (get_log_file_num() <= 1)
+    while (getLogFileNum() <= 1)
     {
         WriteBatch batch;
         PageId page_id2 = 130;
@@ -1761,7 +1754,6 @@ try
 }
 CATCH
 
-
 TEST_F(PageStorageTest, ReloadConfig)
 try
 {

dbms/src/Storages/Page/tools/PageCtl/CMakeLists.txt

@@ -18,7 +18,3 @@ add_library(page-ctl-lib MainEntry.cpp PageStorageCtlV3.cpp PageStorageCtlV2.cpp
 target_include_directories(page-ctl-lib PUBLIC ${TiFlash_SOURCE_DIR}/libs/libdaemon/include)
 target_link_libraries(page-ctl-lib dbms daemon tiflash-dttool-entry-object clickhouse-server-lib)
 target_compile_options(page-ctl-lib  PRIVATE -Wno-format)
-
-add_executable(page-ctl Main.cpp)
-target_link_libraries(page-ctl page-ctl-lib dbms clickhouse_functions clickhouse-server-lib)
-target_compile_options(page-ctl  PRIVATE -Wno-format)

dbms/src/Storages/Page/tools/PageCtl/PageStorageCtlV3.cpp

@@ -22,8 +22,10 @@
 #include <Storages/Page/V3/PageStorageImpl.h>
 #include <Storages/PathPool.h>
 #include <TestUtils/MockDiskDelegator.h>
+#include <TestUtils/TiFlashTestEnv.h>
 
 #include <boost/program_options.hpp>
+#include <magic_enum.hpp>
 
 namespace DB::PS::V3
 {
@@ -32,20 +34,20 @@ void run_raftstore_proxy_ffi(int argc, const char * const * argv, const DB::Engi
 }
 struct ControlOptions
 {
-    enum DisplayType
+    enum class DisplayType
     {
         DISPLAY_SUMMARY_INFO = 1,
         DISPLAY_DIRECTORY_INFO = 2,
         DISPLAY_BLOBS_INFO = 3,
         CHECK_ALL_DATA_CRC = 4,
+        DISPLAY_WAL_ENTRIES = 5,
     };
 
     std::vector<std::string> paths;
-    int display_mode = DisplayType::DISPLAY_SUMMARY_INFO;
-    UInt64 query_page_id = UINT64_MAX;
-    UInt32 query_blob_id = UINT32_MAX;
-    UInt64 query_ns_id = DB::TEST_NAMESPACE_ID;
-    UInt64 check_page_id = UINT64_MAX;
+    DisplayType mode = DisplayType::DISPLAY_SUMMARY_INFO;
+    UInt64 page_id = UINT64_MAX;
+    UInt32 blob_id = UINT32_MAX;
+    UInt64 namespace_id = DB::TEST_NAMESPACE_ID;
     bool enable_fo_check = true;
     bool is_imitative = true;
     String config_file_path;
@@ -62,15 +64,17 @@ ControlOptions ControlOptions::parse(int argc, char ** argv)
     po::options_description desc("Allowed options");
     desc.add_options()("help,h", "produce help message") //
         ("paths,P", value<std::vector<std::string>>(), "store path(s)") //
-        ("display_mode,D", value<int>()->default_value(1), "Display Mode: 1 is summary information,\n"
-                                                           " 2 is display all of stored page and version chain(will be very long),\n"
-                                                           " 3 is display all blobs(in disk) data distribution. \n"
-                                                           " 4 is check every data is valid.") //
+        ("mode", value<int>()->default_value(1), R"(Display Mode:
+ 1 is summary information
+ 2 is display all of stored page and version chain(will be very long)
+ 3 is display all blobs(in disk) data distribution
+ 4 is check every data is valid
+ 5 is dump entries in WAL log files
+)") //
         ("enable_fo_check,E", value<bool>()->default_value(true), "Also check the evert field offsets. This options only works when `display_mode` is 4.") //
-        ("query_ns_id,N", value<UInt64>()->default_value(DB::TEST_NAMESPACE_ID), "When used `check_page_id`/`query_page_id`/`query_blob_id` to query results. You can specify a namespace id.") //
-        ("check_page_id,C", value<UInt64>()->default_value(UINT64_MAX), "Check a single Page id, display the exception if meet. And also will check the field offsets.") //
-        ("query_page_id,W", value<UInt64>()->default_value(UINT64_MAX), "Query a single Page id, and print its version chain.") //
-        ("query_blob_id,B", value<UInt32>()->default_value(UINT32_MAX), "Query a single Blob id, and print its data distribution.") //
+        ("namespace_id,N", value<UInt64>()->default_value(DB::TEST_NAMESPACE_ID), "When used `page_id`/`blob_id` to query results. You can specify a namespace id.") //
+        ("page_id", value<UInt64>()->default_value(UINT64_MAX), "Query a single Page id, and print its version chain.") //
+        ("blob_id,B", value<UInt32>()->default_value(UINT32_MAX), "Query a single Blob id, and print its data distribution.") //
         ("imitative,I", value<bool>()->default_value(true), "Use imitative context instead. (encryption is not supported in this mode so that no need to set config_file_path)") //
         ("config_file_path", value<std::string>(), "Path to TiFlash config (tiflash.toml).");
 
@@ -97,12 +101,11 @@ ControlOptions ControlOptions::parse(int argc, char ** argv)
         exit(0);
     }
     opt.paths = options["paths"].as<std::vector<std::string>>();
-    opt.display_mode = options["display_mode"].as<int>();
-    opt.query_page_id = options["query_page_id"].as<UInt64>();
-    opt.query_blob_id = options["query_blob_id"].as<UInt32>();
+    auto mode_int = options["mode"].as<int>();
+    opt.page_id = options["page_id"].as<UInt64>();
+    opt.blob_id = options["blob_id"].as<UInt32>();
     opt.enable_fo_check = options["enable_fo_check"].as<bool>();
-    opt.check_page_id = options["check_page_id"].as<UInt64>();
-    opt.query_ns_id = options["query_ns_id"].as<UInt64>();
+    opt.namespace_id = options["namespace_id"].as<UInt64>();
     opt.is_imitative = options["imitative"].as<bool>();
     if (opt.is_imitative && options.count("config_file_path") != 0)
     {
@@ -119,12 +122,16 @@ ControlOptions ControlOptions::parse(int argc, char ** argv)
         opt.config_file_path = options["config_file_path"].as<std::string>();
     }
 
-    if (opt.display_mode < DisplayType::DISPLAY_SUMMARY_INFO || opt.display_mode > DisplayType::CHECK_ALL_DATA_CRC)
+    if (auto mode = magic_enum::enum_cast<DisplayType>(mode_int); !mode)
     {
-        std::cerr << "Invalid display mode: " << opt.display_mode << std::endl;
+        std::cerr << "Invalid display mode: " << mode_int << std::endl;
         std::cerr << desc << std::endl;
         exit(0);
     }
+    else
+    {
+        opt.mode = mode.value();
+    }
 
     return opt;
 }
@@ -172,49 +179,59 @@ class PageStorageControlV3
             delegator = std::make_shared<DB::tests::MockDiskDelegatorMulti>(options.paths);
         }
 
-        FileProviderPtr file_provider_ptr;
+        FileProviderPtr provider;
         if (options.is_imitative)
         {
             auto key_manager = std::make_shared<DB::MockKeyManager>(false);
-            file_provider_ptr = std::make_shared<DB::FileProvider>(key_manager, false);
+            provider = std::make_shared<DB::FileProvider>(key_manager, false);
         }
         else
         {
-            file_provider_ptr = context.getFileProvider();
+            provider = context.getFileProvider();
         }
-        BlobConfig blob_config;
 
+        constexpr static std::string_view NAME = "PageStorageControlV3";
         PageStorageConfig config;
-        PageStorageImpl ps_v3("PageStorageControlV3", delegator, config, file_provider_ptr);
-        ps_v3.restore();
-        PageDirectory::MVCCMapType & mvcc_table_directory = ps_v3.page_directory->mvcc_table_directory;
+        if (options.mode == ControlOptions::DisplayType::DISPLAY_WAL_ENTRIES)
+        {
+            // Only restore the PageDirectory
+            PageDirectoryFactory factory;
+            factory.dump_entries = true;
+            factory.create(String(NAME), provider, delegator, WALConfig::from(config));
+            return 0;
+        }
+
+        // Other display mode need to restore ps instance
+        PageStorageImpl ps(String(NAME), delegator, config, provider);
+        ps.restore();
+        PageDirectory::MVCCMapType & mvcc_table_directory = ps.page_directory->mvcc_table_directory;
 
-        switch (options.display_mode)
+        switch (options.mode)
         {
         case ControlOptions::DisplayType::DISPLAY_SUMMARY_INFO:
         {
-            std::cout << getSummaryInfo(mvcc_table_directory, ps_v3.blob_store) << std::endl;
+            std::cout << getSummaryInfo(mvcc_table_directory, ps.blob_store) << std::endl;
             break;
         }
         case ControlOptions::DisplayType::DISPLAY_DIRECTORY_INFO:
         {
-            std::cout << getDirectoryInfo(mvcc_table_directory, options.query_ns_id, options.query_page_id) << std::endl;
+            std::cout << getDirectoryInfo(mvcc_table_directory, options.namespace_id, options.page_id) << std::endl;
             break;
         }
         case ControlOptions::DisplayType::DISPLAY_BLOBS_INFO:
         {
-            std::cout << getBlobsInfo(ps_v3.blob_store, options.query_blob_id) << std::endl;
+            std::cout << getBlobsInfo(ps.blob_store, options.blob_id) << std::endl;
             break;
         }
         case ControlOptions::DisplayType::CHECK_ALL_DATA_CRC:
         {
-            if (options.check_page_id != UINT64_MAX)
+            if (options.page_id != UINT64_MAX)
             {
-                std::cout << checkSinglePage(mvcc_table_directory, ps_v3.blob_store, options.query_ns_id, options.check_page_id) << std::endl;
+                std::cout << checkSinglePage(mvcc_table_directory, ps.blob_store, options.namespace_id, options.page_id) << std::endl;
             }
             else
             {
-                std::cout << checkAllDataCrc(mvcc_table_directory, ps_v3.blob_store, options.enable_fo_check) << std::endl;
+                std::cout << checkAllDataCrc(mvcc_table_directory, ps.blob_store, options.enable_fo_check) << std::endl;
             }
             break;
         }
@@ -507,7 +524,7 @@ class PageStorageControlV3
         {
             error_msg.fmtAppend("id: {}, sequence: {}, epoch: {} \n", internal_id, versioned.sequence, versioned.epoch);
         }
-        error_msg.append("Please use `--query_table_id` + `--check_page_id` to get the more error info.");
+        error_msg.append("Please use `--query_table_id` + `--page_id` to get the more error info.");
 
         return error_msg.toString();
     }
@@ -523,6 +540,7 @@ using namespace DB::PS::V3;
 
 void pageStorageV3CtlEntry(int argc, char ** argv)
 {
+    DB::tests::TiFlashTestEnv::setupLogger();
     const auto & options = ControlOptions::parse(argc, argv);
     PageStorageControlV3(options).run();
 }