From 74fbebc624d5a40caee28f867ba4adc2f50e3c8b Mon Sep 17 00:00:00 2001 From: JaySon Date: Tue, 1 Nov 2022 17:46:00 +0800 Subject: [PATCH] tool: Show the entries from wal log files (#6224) close pingcap/tiflash#5568 --- dbms/src/Storages/Page/Config.h | 2 +- dbms/src/Storages/Page/V3/PageDirectory.cpp | 6 +- .../Storages/Page/V3/PageDirectoryFactory.cpp | 2 + .../Storages/Page/V3/PageDirectoryFactory.h | 4 + .../Page/V3/tests/gtest_page_storage.cpp | 26 ++---- .../Page/tools/PageCtl/CMakeLists.txt | 4 - dbms/src/Storages/Page/tools/PageCtl/Main.cpp | 20 ----- .../Page/tools/PageCtl/PageStorageCtlV3.cpp | 90 +++++++++++-------- 8 files changed, 73 insertions(+), 81 deletions(-) delete mode 100644 dbms/src/Storages/Page/tools/PageCtl/Main.cpp diff --git a/dbms/src/Storages/Page/Config.h b/dbms/src/Storages/Page/Config.h index 21d50d7fefe..cf10d502e12 100644 --- a/dbms/src/Storages/Page/Config.h +++ b/dbms/src/Storages/Page/Config.h @@ -145,7 +145,7 @@ struct PageStorageConfig "PageStorageConfig V3 {{" "blob_file_limit_size: {}, blob_spacemap_type: {}, " "blob_cached_fd_size: {}, blob_heavy_gc_valid_rate: {:.3f}, blob_block_alignment_bytes: {}, " - "wal_roll_size: {}, wal_max_persisted_log_files: {}}}", + "wal_roll_size: {}, wal_max_persisted_log_files: {}}}", blob_file_limit_size.get(), blob_spacemap_type.get(), blob_cached_fd_size.get(), diff --git a/dbms/src/Storages/Page/V3/PageDirectory.cpp b/dbms/src/Storages/Page/V3/PageDirectory.cpp index be0fe39e3fd..46a719c2f84 100644 --- a/dbms/src/Storages/Page/V3/PageDirectory.cpp +++ b/dbms/src/Storages/Page/V3/PageDirectory.cpp @@ -478,7 +478,7 @@ PageSize VersionedPageEntries::getEntriesByBlobIds( auto page_lock = acquireLock(); if (type == EditRecordType::VAR_ENTRY) { - for (const auto & [versioned_type, entry_or_del] : entries) + for (const auto & [ver, entry_or_del] : entries) { if (!entry_or_del.isEntry()) { @@ -488,7 +488,7 @@ PageSize VersionedPageEntries::getEntriesByBlobIds( const auto & entry = entry_or_del.entry; if (blob_ids.count(entry.file_id) > 0) { - blob_versioned_entries[entry.file_id].emplace_back(page_id, versioned_type, entry); + blob_versioned_entries[entry.file_id].emplace_back(page_id, ver, entry); total_entries_size += entry.size; } } @@ -1286,7 +1286,7 @@ PageDirectory::getEntriesByBlobIds(const std::vector & blob_ids) con } } - LOG_INFO(log, "Get entries by Blob ids done. [total_page_size={}] [total_page_nums={}]", // + LOG_INFO(log, "Get entries by blob ids done. [total_page_size={}] [total_page_nums={}]", // total_page_size, // total_page_nums); return std::make_pair(std::move(blob_versioned_entries), total_page_size); diff --git a/dbms/src/Storages/Page/V3/PageDirectoryFactory.cpp b/dbms/src/Storages/Page/V3/PageDirectoryFactory.cpp index 8f07a91b4a2..307d732ce79 100644 --- a/dbms/src/Storages/Page/V3/PageDirectoryFactory.cpp +++ b/dbms/src/Storages/Page/V3/PageDirectoryFactory.cpp @@ -121,6 +121,8 @@ void PageDirectoryFactory::loadEdit(const PageDirectoryPtr & dir, const PageEntr if (max_applied_ver < r.version) max_applied_ver = r.version; + if (dump_entries) + LOG_INFO(Logger::get(), PageEntriesEdit::toDebugString(r)); applyRecord(dir, r); } } diff --git a/dbms/src/Storages/Page/V3/PageDirectoryFactory.h b/dbms/src/Storages/Page/V3/PageDirectoryFactory.h index bca9c8c905d..0252729ad20 100644 --- a/dbms/src/Storages/Page/V3/PageDirectoryFactory.h +++ b/dbms/src/Storages/Page/V3/PageDirectoryFactory.h @@ -69,6 +69,10 @@ class PageDirectoryFactory const PageEntriesEdit::EditRecord & r); BlobStats * blob_stats = nullptr; + + // For debug tool + friend class PageStorageControlV3; + bool dump_entries = false; }; } // namespace PS::V3 diff --git a/dbms/src/Storages/Page/V3/tests/gtest_page_storage.cpp b/dbms/src/Storages/Page/V3/tests/gtest_page_storage.cpp index a919f19a21d..9a63d28a126 100644 --- a/dbms/src/Storages/Page/V3/tests/gtest_page_storage.cpp +++ b/dbms/src/Storages/Page/V3/tests/gtest_page_storage.cpp @@ -55,6 +55,7 @@ class PageStorageTest : public DB::base::TiFlashStorageTestBasic void SetUp() override { TiFlashStorageTestBasic::SetUp(); + log = Logger::get(); auto path = getTemporaryPath(); createIfNotExist(path); file_provider = DB::tests::TiFlashTestEnv::getContext().getFileProvider(); @@ -72,6 +73,11 @@ class PageStorageTest : public DB::base::TiFlashStorageTestBasic return storage; } + size_t getLogFileNum() + { + auto log_files = WALStoreReader::listAllFiles(delegator, log); + return log_files.size(); + } protected: FileProviderPtr file_provider; @@ -80,10 +86,7 @@ class PageStorageTest : public DB::base::TiFlashStorageTestBasic PageStorageConfig config; std::shared_ptr page_storage; - std::list snapshots_holder; - size_t fixed_test_buff_size = 1024; - - size_t epoch_offset = 0; + LoggerPtr log; }; TEST_F(PageStorageTest, WriteRead) @@ -1646,13 +1649,8 @@ try page_storage->write(std::move(batch)); } - auto get_log_file_num = [&]() { - auto log_files = WALStoreReader::listAllFiles(delegator, Logger::get()); - return log_files.size(); - }; - // write until there are more than one wal file - while (get_log_file_num() <= 1) + while (getLogFileNum() <= 1) { WriteBatch batch; PageId page_id1 = 130; @@ -1727,13 +1725,8 @@ try page_storage->write(std::move(batch)); } - auto get_log_file_num = [&]() { - auto log_files = WALStoreReader::listAllFiles(delegator, Logger::get()); - return log_files.size(); - }; - // write until there are more than one wal file - while (get_log_file_num() <= 1) + while (getLogFileNum() <= 1) { WriteBatch batch; PageId page_id2 = 130; @@ -1761,7 +1754,6 @@ try } CATCH - TEST_F(PageStorageTest, ReloadConfig) try { diff --git a/dbms/src/Storages/Page/tools/PageCtl/CMakeLists.txt b/dbms/src/Storages/Page/tools/PageCtl/CMakeLists.txt index 576b5e07a0f..d9b0ae34bf0 100644 --- a/dbms/src/Storages/Page/tools/PageCtl/CMakeLists.txt +++ b/dbms/src/Storages/Page/tools/PageCtl/CMakeLists.txt @@ -18,7 +18,3 @@ add_library(page-ctl-lib MainEntry.cpp PageStorageCtlV3.cpp PageStorageCtlV2.cpp target_include_directories(page-ctl-lib PUBLIC ${TiFlash_SOURCE_DIR}/libs/libdaemon/include) target_link_libraries(page-ctl-lib dbms daemon tiflash-dttool-entry-object clickhouse-server-lib) target_compile_options(page-ctl-lib PRIVATE -Wno-format) - -add_executable(page-ctl Main.cpp) -target_link_libraries(page-ctl page-ctl-lib dbms clickhouse_functions clickhouse-server-lib) -target_compile_options(page-ctl PRIVATE -Wno-format) diff --git a/dbms/src/Storages/Page/tools/PageCtl/Main.cpp b/dbms/src/Storages/Page/tools/PageCtl/Main.cpp deleted file mode 100644 index ae9901ec864..00000000000 --- a/dbms/src/Storages/Page/tools/PageCtl/Main.cpp +++ /dev/null @@ -1,20 +0,0 @@ -// Copyright 2022 PingCAP, Ltd. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include - -int main(int argc, char ** argv) -{ - return DB::PageStorageCtl::mainEntry(argc, argv); -} diff --git a/dbms/src/Storages/Page/tools/PageCtl/PageStorageCtlV3.cpp b/dbms/src/Storages/Page/tools/PageCtl/PageStorageCtlV3.cpp index fec3d7bca94..92d3f91bfab 100644 --- a/dbms/src/Storages/Page/tools/PageCtl/PageStorageCtlV3.cpp +++ b/dbms/src/Storages/Page/tools/PageCtl/PageStorageCtlV3.cpp @@ -22,8 +22,10 @@ #include #include #include +#include #include +#include namespace DB::PS::V3 { @@ -32,20 +34,20 @@ void run_raftstore_proxy_ffi(int argc, const char * const * argv, const DB::Engi } struct ControlOptions { - enum DisplayType + enum class DisplayType { DISPLAY_SUMMARY_INFO = 1, DISPLAY_DIRECTORY_INFO = 2, DISPLAY_BLOBS_INFO = 3, CHECK_ALL_DATA_CRC = 4, + DISPLAY_WAL_ENTRIES = 5, }; std::vector paths; - int display_mode = DisplayType::DISPLAY_SUMMARY_INFO; - UInt64 query_page_id = UINT64_MAX; - UInt32 query_blob_id = UINT32_MAX; - UInt64 query_ns_id = DB::TEST_NAMESPACE_ID; - UInt64 check_page_id = UINT64_MAX; + DisplayType mode = DisplayType::DISPLAY_SUMMARY_INFO; + UInt64 page_id = UINT64_MAX; + UInt32 blob_id = UINT32_MAX; + UInt64 namespace_id = DB::TEST_NAMESPACE_ID; bool enable_fo_check = true; bool is_imitative = true; String config_file_path; @@ -62,15 +64,17 @@ ControlOptions ControlOptions::parse(int argc, char ** argv) po::options_description desc("Allowed options"); desc.add_options()("help,h", "produce help message") // ("paths,P", value>(), "store path(s)") // - ("display_mode,D", value()->default_value(1), "Display Mode: 1 is summary information,\n" - " 2 is display all of stored page and version chain(will be very long),\n" - " 3 is display all blobs(in disk) data distribution. \n" - " 4 is check every data is valid.") // + ("mode", value()->default_value(1), R"(Display Mode: + 1 is summary information + 2 is display all of stored page and version chain(will be very long) + 3 is display all blobs(in disk) data distribution + 4 is check every data is valid + 5 is dump entries in WAL log files +)") // ("enable_fo_check,E", value()->default_value(true), "Also check the evert field offsets. This options only works when `display_mode` is 4.") // - ("query_ns_id,N", value()->default_value(DB::TEST_NAMESPACE_ID), "When used `check_page_id`/`query_page_id`/`query_blob_id` to query results. You can specify a namespace id.") // - ("check_page_id,C", value()->default_value(UINT64_MAX), "Check a single Page id, display the exception if meet. And also will check the field offsets.") // - ("query_page_id,W", value()->default_value(UINT64_MAX), "Query a single Page id, and print its version chain.") // - ("query_blob_id,B", value()->default_value(UINT32_MAX), "Query a single Blob id, and print its data distribution.") // + ("namespace_id,N", value()->default_value(DB::TEST_NAMESPACE_ID), "When used `page_id`/`blob_id` to query results. You can specify a namespace id.") // + ("page_id", value()->default_value(UINT64_MAX), "Query a single Page id, and print its version chain.") // + ("blob_id,B", value()->default_value(UINT32_MAX), "Query a single Blob id, and print its data distribution.") // ("imitative,I", value()->default_value(true), "Use imitative context instead. (encryption is not supported in this mode so that no need to set config_file_path)") // ("config_file_path", value(), "Path to TiFlash config (tiflash.toml)."); @@ -97,12 +101,11 @@ ControlOptions ControlOptions::parse(int argc, char ** argv) exit(0); } opt.paths = options["paths"].as>(); - opt.display_mode = options["display_mode"].as(); - opt.query_page_id = options["query_page_id"].as(); - opt.query_blob_id = options["query_blob_id"].as(); + auto mode_int = options["mode"].as(); + opt.page_id = options["page_id"].as(); + opt.blob_id = options["blob_id"].as(); opt.enable_fo_check = options["enable_fo_check"].as(); - opt.check_page_id = options["check_page_id"].as(); - opt.query_ns_id = options["query_ns_id"].as(); + opt.namespace_id = options["namespace_id"].as(); opt.is_imitative = options["imitative"].as(); if (opt.is_imitative && options.count("config_file_path") != 0) { @@ -119,12 +122,16 @@ ControlOptions ControlOptions::parse(int argc, char ** argv) opt.config_file_path = options["config_file_path"].as(); } - if (opt.display_mode < DisplayType::DISPLAY_SUMMARY_INFO || opt.display_mode > DisplayType::CHECK_ALL_DATA_CRC) + if (auto mode = magic_enum::enum_cast(mode_int); !mode) { - std::cerr << "Invalid display mode: " << opt.display_mode << std::endl; + std::cerr << "Invalid display mode: " << mode_int << std::endl; std::cerr << desc << std::endl; exit(0); } + else + { + opt.mode = mode.value(); + } return opt; } @@ -172,49 +179,59 @@ class PageStorageControlV3 delegator = std::make_shared(options.paths); } - FileProviderPtr file_provider_ptr; + FileProviderPtr provider; if (options.is_imitative) { auto key_manager = std::make_shared(false); - file_provider_ptr = std::make_shared(key_manager, false); + provider = std::make_shared(key_manager, false); } else { - file_provider_ptr = context.getFileProvider(); + provider = context.getFileProvider(); } - BlobConfig blob_config; + constexpr static std::string_view NAME = "PageStorageControlV3"; PageStorageConfig config; - PageStorageImpl ps_v3("PageStorageControlV3", delegator, config, file_provider_ptr); - ps_v3.restore(); - PageDirectory::MVCCMapType & mvcc_table_directory = ps_v3.page_directory->mvcc_table_directory; + if (options.mode == ControlOptions::DisplayType::DISPLAY_WAL_ENTRIES) + { + // Only restore the PageDirectory + PageDirectoryFactory factory; + factory.dump_entries = true; + factory.create(String(NAME), provider, delegator, WALConfig::from(config)); + return 0; + } + + // Other display mode need to restore ps instance + PageStorageImpl ps(String(NAME), delegator, config, provider); + ps.restore(); + PageDirectory::MVCCMapType & mvcc_table_directory = ps.page_directory->mvcc_table_directory; - switch (options.display_mode) + switch (options.mode) { case ControlOptions::DisplayType::DISPLAY_SUMMARY_INFO: { - std::cout << getSummaryInfo(mvcc_table_directory, ps_v3.blob_store) << std::endl; + std::cout << getSummaryInfo(mvcc_table_directory, ps.blob_store) << std::endl; break; } case ControlOptions::DisplayType::DISPLAY_DIRECTORY_INFO: { - std::cout << getDirectoryInfo(mvcc_table_directory, options.query_ns_id, options.query_page_id) << std::endl; + std::cout << getDirectoryInfo(mvcc_table_directory, options.namespace_id, options.page_id) << std::endl; break; } case ControlOptions::DisplayType::DISPLAY_BLOBS_INFO: { - std::cout << getBlobsInfo(ps_v3.blob_store, options.query_blob_id) << std::endl; + std::cout << getBlobsInfo(ps.blob_store, options.blob_id) << std::endl; break; } case ControlOptions::DisplayType::CHECK_ALL_DATA_CRC: { - if (options.check_page_id != UINT64_MAX) + if (options.page_id != UINT64_MAX) { - std::cout << checkSinglePage(mvcc_table_directory, ps_v3.blob_store, options.query_ns_id, options.check_page_id) << std::endl; + std::cout << checkSinglePage(mvcc_table_directory, ps.blob_store, options.namespace_id, options.page_id) << std::endl; } else { - std::cout << checkAllDataCrc(mvcc_table_directory, ps_v3.blob_store, options.enable_fo_check) << std::endl; + std::cout << checkAllDataCrc(mvcc_table_directory, ps.blob_store, options.enable_fo_check) << std::endl; } break; } @@ -507,7 +524,7 @@ class PageStorageControlV3 { error_msg.fmtAppend("id: {}, sequence: {}, epoch: {} \n", internal_id, versioned.sequence, versioned.epoch); } - error_msg.append("Please use `--query_table_id` + `--check_page_id` to get the more error info."); + error_msg.append("Please use `--query_table_id` + `--page_id` to get the more error info."); return error_msg.toString(); } @@ -523,6 +540,7 @@ using namespace DB::PS::V3; void pageStorageV3CtlEntry(int argc, char ** argv) { + DB::tests::TiFlashTestEnv::setupLogger(); const auto & options = ControlOptions::parse(argc, argv); PageStorageControlV3(options).run(); }