diff --git a/dbms/src/Debug/dbgFuncMisc.cpp b/dbms/src/Debug/dbgFuncMisc.cpp index bb907604654..5ee1e1f1e96 100644 --- a/dbms/src/Debug/dbgFuncMisc.cpp +++ b/dbms/src/Debug/dbgFuncMisc.cpp @@ -17,7 +17,8 @@ #include #include #include -#include +#include +#include #include #include diff --git a/dbms/src/Interpreters/AsynchronousMetrics.cpp b/dbms/src/Interpreters/AsynchronousMetrics.cpp index 95d3e2d6611..27009efd6a6 100644 --- a/dbms/src/Interpreters/AsynchronousMetrics.cpp +++ b/dbms/src/Interpreters/AsynchronousMetrics.cpp @@ -25,7 +25,8 @@ #include #include #include -#include +#include +#include #include #include #include diff --git a/dbms/src/Interpreters/Context.cpp b/dbms/src/Interpreters/Context.cpp index ba923eeaf05..8b5a73ac7ca 100644 --- a/dbms/src/Interpreters/Context.cpp +++ b/dbms/src/Interpreters/Context.cpp @@ -57,7 +57,8 @@ #include #include #include -#include +#include +#include #include #include #include diff --git a/dbms/src/Server/DTTool/DTToolBench.cpp b/dbms/src/Server/DTTool/DTToolBench.cpp index de965c2b9c6..0565e5622d0 100644 --- a/dbms/src/Server/DTTool/DTToolBench.cpp +++ b/dbms/src/Server/DTTool/DTToolBench.cpp @@ -25,7 +25,7 @@ #include #include #include -#include +#include #include #include #include diff --git a/dbms/src/Server/tests/gtest_dttool.cpp b/dbms/src/Server/tests/gtest_dttool.cpp index 844cea07797..36d6cc4babb 100644 --- a/dbms/src/Server/tests/gtest_dttool.cpp +++ b/dbms/src/Server/tests/gtest_dttool.cpp @@ -19,7 +19,7 @@ #include #include #include -#include +#include #include #include #include diff --git a/dbms/src/Server/tests/gtest_server_config.cpp b/dbms/src/Server/tests/gtest_server_config.cpp index e223cb2f7cd..05497eeaa24 100644 --- a/dbms/src/Server/tests/gtest_server_config.cpp +++ b/dbms/src/Server/tests/gtest_server_config.cpp @@ -28,7 +28,8 @@ #include #include #include -#include +#include +#include #include #include #include diff --git a/dbms/src/Storages/DeltaMerge/CMakeLists.txt b/dbms/src/Storages/DeltaMerge/CMakeLists.txt index 75be70dc8fb..2f7326f0bae 100644 --- a/dbms/src/Storages/DeltaMerge/CMakeLists.txt +++ b/dbms/src/Storages/DeltaMerge/CMakeLists.txt @@ -30,6 +30,7 @@ add_headers_and_sources(delta_merge ./ReadThread) add_headers_and_sources(delta_merge ./Remote) add_headers_and_sources(delta_merge ./Remote/DataStore) add_headers_and_sources(delta_merge ./Decode) +add_headers_and_sources(delta_merge ./StoragePool) add_library(delta_merge ${delta_merge_headers} ${delta_merge_sources}) target_link_libraries(delta_merge PRIVATE dbms page) diff --git a/dbms/src/Storages/DeltaMerge/ColumnFile/ColumnFile.h b/dbms/src/Storages/DeltaMerge/ColumnFile/ColumnFile.h index ccd87d33f24..5f70867584f 100644 --- a/dbms/src/Storages/DeltaMerge/ColumnFile/ColumnFile.h +++ b/dbms/src/Storages/DeltaMerge/ColumnFile/ColumnFile.h @@ -20,7 +20,7 @@ #include #include #include -#include +#include #include namespace DB @@ -130,7 +130,8 @@ class ColumnFile virtual ColumnFileReaderPtr getReader( const DMContext & context, const IColumnFileDataProviderPtr & data_provider, - const ColumnDefinesPtr & col_defs) const = 0; + const ColumnDefinesPtr & col_defs) const + = 0; /// Note: Only ColumnFileInMemory can be appendable. Other ColumnFiles (i.e. ColumnFilePersisted) have /// been persisted in the disk and their data will be immutable. diff --git a/dbms/src/Storages/DeltaMerge/ColumnFile/ColumnFileDataProvider.cpp b/dbms/src/Storages/DeltaMerge/ColumnFile/ColumnFileDataProvider.cpp index de4fbb743d6..c8034ed866f 100644 --- a/dbms/src/Storages/DeltaMerge/ColumnFile/ColumnFileDataProvider.cpp +++ b/dbms/src/Storages/DeltaMerge/ColumnFile/ColumnFileDataProvider.cpp @@ -13,7 +13,7 @@ // limitations under the License. #include -#include +#include #include namespace DB::DM diff --git a/dbms/src/Storages/DeltaMerge/ColumnFile/ColumnFileDataProvider.h b/dbms/src/Storages/DeltaMerge/ColumnFile/ColumnFileDataProvider.h index b65bd04c61e..66a6f7586d1 100644 --- a/dbms/src/Storages/DeltaMerge/ColumnFile/ColumnFileDataProvider.h +++ b/dbms/src/Storages/DeltaMerge/ColumnFile/ColumnFileDataProvider.h @@ -15,7 +15,7 @@ #pragma once #include -#include +#include #include #include diff --git a/dbms/src/Storages/DeltaMerge/Delta/Snapshot.cpp b/dbms/src/Storages/DeltaMerge/Delta/Snapshot.cpp index 8650686d1dc..a6b7151aa39 100644 --- a/dbms/src/Storages/DeltaMerge/Delta/Snapshot.cpp +++ b/dbms/src/Storages/DeltaMerge/Delta/Snapshot.cpp @@ -17,7 +17,7 @@ #include #include #include -#include +#include #include namespace DB::DM diff --git a/dbms/src/Storages/DeltaMerge/DeltaMergeStore_InternalBg.cpp b/dbms/src/Storages/DeltaMerge/DeltaMergeStore_InternalBg.cpp index 5a0100201bd..6b21fefe326 100644 --- a/dbms/src/Storages/DeltaMerge/DeltaMergeStore_InternalBg.cpp +++ b/dbms/src/Storages/DeltaMerge/DeltaMergeStore_InternalBg.cpp @@ -21,7 +21,7 @@ #include #include #include -#include +#include #include #include #include diff --git a/dbms/src/Storages/DeltaMerge/DeltaMergeStore_Statistics.cpp b/dbms/src/Storages/DeltaMerge/DeltaMergeStore_Statistics.cpp index c50d7b33767..411d09a9f22 100644 --- a/dbms/src/Storages/DeltaMerge/DeltaMergeStore_Statistics.cpp +++ b/dbms/src/Storages/DeltaMerge/DeltaMergeStore_Statistics.cpp @@ -14,7 +14,7 @@ #include #include -#include +#include #include namespace DB diff --git a/dbms/src/Storages/DeltaMerge/Segment.cpp b/dbms/src/Storages/DeltaMerge/Segment.cpp index 0f275e59697..fc89e134428 100644 --- a/dbms/src/Storages/DeltaMerge/Segment.cpp +++ b/dbms/src/Storages/DeltaMerge/Segment.cpp @@ -46,7 +46,7 @@ #include #include #include -#include +#include #include #include #include diff --git a/dbms/src/Storages/DeltaMerge/StableValueSpace.cpp b/dbms/src/Storages/DeltaMerge/StableValueSpace.cpp index 2f0fceebfc3..279e4b85c92 100644 --- a/dbms/src/Storages/DeltaMerge/StableValueSpace.cpp +++ b/dbms/src/Storages/DeltaMerge/StableValueSpace.cpp @@ -23,7 +23,7 @@ #include #include #include -#include +#include #include #include #include diff --git a/dbms/src/Storages/DeltaMerge/StoragePool/GlobalStoragePool.cpp b/dbms/src/Storages/DeltaMerge/StoragePool/GlobalStoragePool.cpp new file mode 100644 index 00000000000..a8389926c51 --- /dev/null +++ b/dbms/src/Storages/DeltaMerge/StoragePool/GlobalStoragePool.cpp @@ -0,0 +1,116 @@ +// Copyright 2024 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include +#include +#include +#include +#include +#include + +namespace DB::DM +{ + +GlobalStoragePool::GlobalStoragePool(const PathPool & path_pool, Context & global_ctx, const Settings & settings) + : log_storage(PageStorage::create( + "__global__.log", + path_pool.getPSDiskDelegatorGlobalMulti(PathPool::log_path_prefix), + extractConfig(settings, StorageType::Log), + global_ctx.getFileProvider(), + global_ctx, + true)) + , data_storage(PageStorage::create( + "__global__.data", + path_pool.getPSDiskDelegatorGlobalMulti(PathPool::data_path_prefix), + extractConfig(settings, StorageType::Data), + global_ctx.getFileProvider(), + global_ctx, + true)) + , meta_storage(PageStorage::create( + "__global__.meta", + path_pool.getPSDiskDelegatorGlobalMulti(PathPool::meta_path_prefix), + extractConfig(settings, StorageType::Meta), + global_ctx.getFileProvider(), + global_ctx, + true)) + , global_context(global_ctx) +{} + + +GlobalStoragePool::~GlobalStoragePool() +{ + shutdown(); +} + +void GlobalStoragePool::restore() +{ + log_storage->restore(); + data_storage->restore(); + meta_storage->restore(); + + gc_handle = global_context.getBackgroundPool().addTask( + [this] { return this->gc(global_context.getSettingsRef()); }, + false); +} + +void GlobalStoragePool::shutdown() +{ + if (gc_handle) + { + global_context.getBackgroundPool().removeTask(gc_handle); + gc_handle = {}; + } +} + +FileUsageStatistics GlobalStoragePool::getLogFileUsage() const +{ + return log_storage->getFileUsageStatistics(); +} + +bool GlobalStoragePool::gc() +{ + return gc(global_context.getSettingsRef(), /*immediately=*/true, DELTA_MERGE_GC_PERIOD); +} + +bool GlobalStoragePool::gc(const Settings & settings, bool immediately, const Seconds & try_gc_period) +{ + Timepoint now = Clock::now(); + if (!immediately) + { + // No need lock + if (now < (last_try_gc_time.load() + try_gc_period)) + return false; + } + + last_try_gc_time = now; + + bool done_anything = false; + auto write_limiter = global_context.getWriteLimiter(); + auto read_limiter = global_context.getReadLimiter(); + auto config = extractConfig(settings, StorageType::Meta); + meta_storage->reloadSettings(config); + done_anything |= meta_storage->gc(/*not_skip*/ false, write_limiter, read_limiter); + + config = extractConfig(settings, StorageType::Data); + data_storage->reloadSettings(config); + done_anything |= data_storage->gc(/*not_skip*/ false, write_limiter, read_limiter); + + config = extractConfig(settings, StorageType::Log); + log_storage->reloadSettings(config); + done_anything |= log_storage->gc(/*not_skip*/ false, write_limiter, read_limiter); + + return done_anything; +} + +} // namespace DB::DM diff --git a/dbms/src/Storages/DeltaMerge/StoragePool/GlobalStoragePool.h b/dbms/src/Storages/DeltaMerge/StoragePool/GlobalStoragePool.h new file mode 100644 index 00000000000..b21116a413b --- /dev/null +++ b/dbms/src/Storages/DeltaMerge/StoragePool/GlobalStoragePool.h @@ -0,0 +1,82 @@ +// Copyright 2024 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#pragma once + +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +namespace DB +{ +class WriteLimiter; +using WriteLimiterPtr = std::shared_ptr; +class ReadLimiter; +using ReadLimiterPtr = std::shared_ptr; + +struct Settings; +class PathPool; +class StableDiskDelegator; +class AsynchronousMetrics; +} // namespace DB + +namespace DB::DM +{ + +class GlobalStoragePool : private boost::noncopyable +{ +public: + using Clock = std::chrono::system_clock; + using Timepoint = Clock::time_point; + using Seconds = std::chrono::seconds; + + GlobalStoragePool(const PathPool & path_pool, Context & global_ctx, const Settings & settings); + + ~GlobalStoragePool(); + + void restore(); + + void shutdown(); + + friend class StoragePool; + friend class ::DB::AsynchronousMetrics; + + // GC immediately + // Only used on dbgFuncMisc + bool gc(); + + FileUsageStatistics getLogFileUsage() const; + +private: + bool gc(const Settings & settings, bool immediately = false, const Seconds & try_gc_period = DELTA_MERGE_GC_PERIOD); + +private: + PageStoragePtr log_storage; + PageStoragePtr data_storage; + PageStoragePtr meta_storage; + + std::atomic last_try_gc_time = Clock::now(); + + Context & global_context; + BackgroundProcessingPool::TaskHandle gc_handle; +}; + +} // namespace DB::DM diff --git a/dbms/src/Storages/DeltaMerge/StoragePool.cpp b/dbms/src/Storages/DeltaMerge/StoragePool/StoragePool.cpp similarity index 85% rename from dbms/src/Storages/DeltaMerge/StoragePool.cpp rename to dbms/src/Storages/DeltaMerge/StoragePool/StoragePool.cpp index e3f754db808..c522bcdfbcc 100644 --- a/dbms/src/Storages/DeltaMerge/StoragePool.cpp +++ b/dbms/src/Storages/DeltaMerge/StoragePool/StoragePool.cpp @@ -1,4 +1,4 @@ -// Copyright 2023 PingCAP, Inc. +// Copyright 2024 PingCAP, Inc. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -16,7 +16,9 @@ #include #include #include -#include +#include +#include +#include #include #include #include @@ -48,133 +50,10 @@ namespace FailPoints { extern const char force_set_dtfile_exist_when_acquire_id[]; } // namespace FailPoints +} // namespace DB -namespace DM -{ - -PageStorageConfig extractConfig(const Settings & settings, StorageType subtype) -{ -#define SET_CONFIG(NAME) \ - config.num_write_slots = settings.dt_storage_pool_##NAME##_write_slots; \ - config.gc_min_files = settings.dt_storage_pool_##NAME##_gc_min_file_num; \ - config.gc_min_bytes = settings.dt_storage_pool_##NAME##_gc_min_bytes; \ - config.gc_min_legacy_num = settings.dt_storage_pool_##NAME##_gc_min_legacy_num; \ - config.gc_max_valid_rate = settings.dt_storage_pool_##NAME##_gc_max_valid_rate; \ - config.blob_heavy_gc_valid_rate = settings.dt_page_gc_threshold; - - PageStorageConfig config = getConfigFromSettings(settings); - - switch (subtype) - { - case StorageType::Log: - SET_CONFIG(log); - break; - case StorageType::Data: - SET_CONFIG(data); - break; - case StorageType::Meta: - SET_CONFIG(meta); - break; - default: - throw Exception( - fmt::format("Unknown subtype in extractConfig: {} ", static_cast(subtype)), - ErrorCodes::LOGICAL_ERROR); - } -#undef SET_CONFIG - - return config; -} - -GlobalStoragePool::GlobalStoragePool(const PathPool & path_pool, Context & global_ctx, const Settings & settings) - : log_storage(PageStorage::create( - "__global__.log", - path_pool.getPSDiskDelegatorGlobalMulti(PathPool::log_path_prefix), - extractConfig(settings, StorageType::Log), - global_ctx.getFileProvider(), - global_ctx, - true)) - , data_storage(PageStorage::create( - "__global__.data", - path_pool.getPSDiskDelegatorGlobalMulti(PathPool::data_path_prefix), - extractConfig(settings, StorageType::Data), - global_ctx.getFileProvider(), - global_ctx, - true)) - , meta_storage(PageStorage::create( - "__global__.meta", - path_pool.getPSDiskDelegatorGlobalMulti(PathPool::meta_path_prefix), - extractConfig(settings, StorageType::Meta), - global_ctx.getFileProvider(), - global_ctx, - true)) - , global_context(global_ctx) -{} - - -GlobalStoragePool::~GlobalStoragePool() -{ - shutdown(); -} - -void GlobalStoragePool::restore() -{ - log_storage->restore(); - data_storage->restore(); - meta_storage->restore(); - - gc_handle = global_context.getBackgroundPool().addTask( - [this] { return this->gc(global_context.getSettingsRef()); }, - false); -} - -void GlobalStoragePool::shutdown() -{ - if (gc_handle) - { - global_context.getBackgroundPool().removeTask(gc_handle); - gc_handle = {}; - } -} - -FileUsageStatistics GlobalStoragePool::getLogFileUsage() const -{ - return log_storage->getFileUsageStatistics(); -} - -bool GlobalStoragePool::gc() -{ - return gc(global_context.getSettingsRef(), /*immediately=*/true, DELTA_MERGE_GC_PERIOD); -} - -bool GlobalStoragePool::gc(const Settings & settings, bool immediately, const Seconds & try_gc_period) +namespace DB::DM { - Timepoint now = Clock::now(); - if (!immediately) - { - // No need lock - if (now < (last_try_gc_time.load() + try_gc_period)) - return false; - } - - last_try_gc_time = now; - - bool done_anything = false; - auto write_limiter = global_context.getWriteLimiter(); - auto read_limiter = global_context.getReadLimiter(); - auto config = extractConfig(settings, StorageType::Meta); - meta_storage->reloadSettings(config); - done_anything |= meta_storage->gc(/*not_skip*/ false, write_limiter, read_limiter); - - config = extractConfig(settings, StorageType::Data); - data_storage->reloadSettings(config); - done_anything |= data_storage->gc(/*not_skip*/ false, write_limiter, read_limiter); - - config = extractConfig(settings, StorageType::Log); - log_storage->reloadSettings(config); - done_anything |= log_storage->gc(/*not_skip*/ false, write_limiter, read_limiter); - - return done_anything; -} StoragePool::StoragePool( Context & global_ctx, @@ -468,9 +347,7 @@ StoragePool::StoragePool( break; } default: - throw Exception( - fmt::format("Unknown PageStorageRunMode {}", static_cast(run_mode)), - ErrorCodes::LOGICAL_ERROR); + throw Exception(ErrorCodes::LOGICAL_ERROR, "Unknown PageStorageRunMode {}", static_cast(run_mode)); } } @@ -478,7 +355,10 @@ void StoragePool::forceTransformMetaV2toV3() { if (unlikely(run_mode != PageStorageRunMode::MIX_MODE)) throw Exception( - fmt::format("Transform meta must run under mix mode [run_mode={}]", static_cast(run_mode))); + ErrorCodes::LOGICAL_ERROR, + "Transform meta must run under mix mode [run_mode={}]", + static_cast(run_mode)); + assert(meta_storage_v2 != nullptr); assert(meta_storage_v3 != nullptr); auto meta_transform_storage_writer = std::make_shared( @@ -515,11 +395,10 @@ void StoragePool::forceTransformMetaV2toV3() if (!page_transform_entry.field_offsets.empty()) { throw Exception( - fmt::format( - "Can't transform meta from V2 to V3, [page_id={}] {}", // - page_transform.page_id, - page_transform_entry.toDebugString()), - ErrorCodes::LOGICAL_ERROR); + ErrorCodes::LOGICAL_ERROR, + "Can't transform meta from V2 to V3, [page_id={}] {}", // + page_transform.page_id, + page_transform_entry.toDebugString()); } write_batch_transform.putPage( @@ -548,7 +427,9 @@ void StoragePool::forceTransformDataV2toV3() { if (unlikely(run_mode != PageStorageRunMode::MIX_MODE)) throw Exception( - fmt::format("Transform meta must run under mix mode [run_mode={}]", static_cast(run_mode))); + ErrorCodes::LOGICAL_ERROR, + "Transform meta must run under mix mode [run_mode={}]", + static_cast(run_mode)); assert(data_storage_v2 != nullptr); assert(data_storage_v3 != nullptr); auto data_transform_storage_writer = std::make_shared( @@ -1050,9 +931,7 @@ inline static PageReaderPtr newReader( snapshot_read ? uni_ps->getSnapshot(tracing_id) : nullptr, read_limiter); default: - throw Exception( - fmt::format("Unknown PageStorageRunMode {}", static_cast(run_mode)), - ErrorCodes::LOGICAL_ERROR); + throw Exception(ErrorCodes::LOGICAL_ERROR, "Unknown PageStorageRunMode {}", static_cast(run_mode)); } } @@ -1100,20 +979,6 @@ PageReaderPtr StoragePool::newDataReader(ReadLimiterPtr read_limiter, bool snaps tracing_id); } -PageReaderPtr StoragePool::newDataReader(ReadLimiterPtr read_limiter, PageStorage::SnapshotPtr & snapshot) -{ - return std::make_shared( - run_mode, - keyspace_id, - StorageType::Data, - ns_id, - data_storage_v2, - data_storage_v3, - uni_ps, - snapshot, - read_limiter); -} - PageReaderPtr StoragePool::newMetaReader(ReadLimiterPtr read_limiter, bool snapshot_read, const String & tracing_id) { return newReader( @@ -1129,19 +994,4 @@ PageReaderPtr StoragePool::newMetaReader(ReadLimiterPtr read_limiter, bool snaps tracing_id); } -PageReaderPtr StoragePool::newMetaReader(ReadLimiterPtr read_limiter, PageStorage::SnapshotPtr & snapshot) -{ - return std::make_shared( - run_mode, - keyspace_id, - StorageType::Meta, - ns_id, - meta_storage_v2, - meta_storage_v3, - uni_ps, - snapshot, - read_limiter); -} - -} // namespace DM -} // namespace DB +} // namespace DB::DM diff --git a/dbms/src/Storages/DeltaMerge/StoragePool.h b/dbms/src/Storages/DeltaMerge/StoragePool/StoragePool.h similarity index 82% rename from dbms/src/Storages/DeltaMerge/StoragePool.h rename to dbms/src/Storages/DeltaMerge/StoragePool/StoragePool.h index 762a0dfcbd5..33bf28af65b 100644 --- a/dbms/src/Storages/DeltaMerge/StoragePool.h +++ b/dbms/src/Storages/DeltaMerge/StoragePool/StoragePool.h @@ -1,4 +1,4 @@ -// Copyright 2023 PingCAP, Inc. +// Copyright 2024 PingCAP, Inc. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -17,7 +17,7 @@ #include #include #include -#include +#include #include #include #include @@ -38,48 +38,10 @@ class StoragePathPool; class PathPool; class StableDiskDelegator; class AsynchronousMetrics; +} // namespace DB -namespace DM -{ -static constexpr std::chrono::seconds DELTA_MERGE_GC_PERIOD(60); - -class GlobalStoragePool : private boost::noncopyable +namespace DB::DM { -public: - using Clock = std::chrono::system_clock; - using Timepoint = Clock::time_point; - using Seconds = std::chrono::seconds; - - GlobalStoragePool(const PathPool & path_pool, Context & global_ctx, const Settings & settings); - - ~GlobalStoragePool(); - - void restore(); - - void shutdown(); - - friend class StoragePool; - friend class ::DB::AsynchronousMetrics; - - // GC immediately - // Only used on dbgFuncMisc - bool gc(); - - FileUsageStatistics getLogFileUsage() const; - -private: - bool gc(const Settings & settings, bool immediately = false, const Seconds & try_gc_period = DELTA_MERGE_GC_PERIOD); - -private: - PageStoragePtr log_storage; - PageStoragePtr data_storage; - PageStoragePtr meta_storage; - - std::atomic last_try_gc_time = Clock::now(); - - Context & global_context; - BackgroundProcessingPool::TaskHandle gc_handle; -}; class StoragePool : private boost::noncopyable { @@ -146,10 +108,8 @@ class StoragePool : private boost::noncopyable PageReaderPtr newLogReader(ReadLimiterPtr read_limiter, PageStorageSnapshotPtr & snapshot); PageReaderPtr newDataReader(ReadLimiterPtr read_limiter, bool snapshot_read, const String & tracing_id); - PageReaderPtr newDataReader(ReadLimiterPtr read_limiter, PageStorageSnapshotPtr & snapshot); PageReaderPtr newMetaReader(ReadLimiterPtr read_limiter, bool snapshot_read, const String & tracing_id); - PageReaderPtr newMetaReader(ReadLimiterPtr read_limiter, PageStorageSnapshotPtr & snapshot); // Register the clean up DMFiles callbacks to PageStorage. // The callbacks will be unregister when `shutdown` is called. @@ -244,5 +204,4 @@ struct StorageSnapshot : private boost::noncopyable }; -} // namespace DM -} // namespace DB +} // namespace DB::DM diff --git a/dbms/src/Storages/DeltaMerge/StoragePool/StoragePoolConfig.h b/dbms/src/Storages/DeltaMerge/StoragePool/StoragePoolConfig.h new file mode 100644 index 00000000000..c4467c5c24f --- /dev/null +++ b/dbms/src/Storages/DeltaMerge/StoragePool/StoragePoolConfig.h @@ -0,0 +1,59 @@ +// Copyright 2024 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#pragma once + +#include +#include +#include +#include + +namespace DB::DM +{ + +inline PageStorageConfig extractConfig(const Settings & settings, StorageType subtype) +{ +#define SET_CONFIG(NAME) \ + config.num_write_slots = settings.dt_storage_pool_##NAME##_write_slots; \ + config.gc_min_files = settings.dt_storage_pool_##NAME##_gc_min_file_num; \ + config.gc_min_bytes = settings.dt_storage_pool_##NAME##_gc_min_bytes; \ + config.gc_min_legacy_num = settings.dt_storage_pool_##NAME##_gc_min_legacy_num; \ + config.gc_max_valid_rate = settings.dt_storage_pool_##NAME##_gc_max_valid_rate; \ + config.blob_heavy_gc_valid_rate = settings.dt_page_gc_threshold; + + PageStorageConfig config = getConfigFromSettings(settings); + + switch (subtype) + { + case StorageType::Log: + SET_CONFIG(log); + break; + case StorageType::Data: + SET_CONFIG(data); + break; + case StorageType::Meta: + SET_CONFIG(meta); + break; + default: + throw Exception( + ErrorCodes::LOGICAL_ERROR, + "Unknown subtype in extractConfig: {} ", + static_cast(subtype)); + } +#undef SET_CONFIG + + return config; +} + +} // namespace DB::DM diff --git a/dbms/src/Storages/DeltaMerge/StoragePool_fwd.h b/dbms/src/Storages/DeltaMerge/StoragePool/StoragePool_fwd.h similarity index 90% rename from dbms/src/Storages/DeltaMerge/StoragePool_fwd.h rename to dbms/src/Storages/DeltaMerge/StoragePool/StoragePool_fwd.h index 156bda060b2..4dc7aa17166 100644 --- a/dbms/src/Storages/DeltaMerge/StoragePool_fwd.h +++ b/dbms/src/Storages/DeltaMerge/StoragePool/StoragePool_fwd.h @@ -1,4 +1,4 @@ -// Copyright 2023 PingCAP, Inc. +// Copyright 2024 PingCAP, Inc. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -28,4 +28,6 @@ using GlobalStoragePoolPtr = std::shared_ptr; struct StorageSnapshot; using StorageSnapshotPtr = std::shared_ptr; +static constexpr std::chrono::seconds DELTA_MERGE_GC_PERIOD(60); + } // namespace DB::DM diff --git a/dbms/src/Storages/DeltaMerge/WriteBatchesImpl.h b/dbms/src/Storages/DeltaMerge/WriteBatchesImpl.h index d33f075336e..1fec5fe7e9e 100644 --- a/dbms/src/Storages/DeltaMerge/WriteBatchesImpl.h +++ b/dbms/src/Storages/DeltaMerge/WriteBatchesImpl.h @@ -15,7 +15,7 @@ #pragma once #include -#include +#include #include #include diff --git a/dbms/src/Storages/DeltaMerge/tests/gtest_dm_column_file.cpp b/dbms/src/Storages/DeltaMerge/tests/gtest_dm_column_file.cpp index fce29783fd8..9f78c030087 100644 --- a/dbms/src/Storages/DeltaMerge/tests/gtest_dm_column_file.cpp +++ b/dbms/src/Storages/DeltaMerge/tests/gtest_dm_column_file.cpp @@ -26,7 +26,7 @@ #include #include #include -#include +#include #include #include #include diff --git a/dbms/src/Storages/DeltaMerge/tests/gtest_dm_delta_merge_store.cpp b/dbms/src/Storages/DeltaMerge/tests/gtest_dm_delta_merge_store.cpp index 14a28b27abc..7cfebd826dc 100644 --- a/dbms/src/Storages/DeltaMerge/tests/gtest_dm_delta_merge_store.cpp +++ b/dbms/src/Storages/DeltaMerge/tests/gtest_dm_delta_merge_store.cpp @@ -24,7 +24,8 @@ #include #include #include -#include +#include +#include #include #include #include diff --git a/dbms/src/Storages/DeltaMerge/tests/gtest_dm_delta_merge_store_fast_add_peer.cpp b/dbms/src/Storages/DeltaMerge/tests/gtest_dm_delta_merge_store_fast_add_peer.cpp index 0e0cc48e75e..fd08d5596ae 100644 --- a/dbms/src/Storages/DeltaMerge/tests/gtest_dm_delta_merge_store_fast_add_peer.cpp +++ b/dbms/src/Storages/DeltaMerge/tests/gtest_dm_delta_merge_store_fast_add_peer.cpp @@ -23,7 +23,7 @@ #include #include #include -#include +#include #include #include #include diff --git a/dbms/src/Storages/DeltaMerge/tests/gtest_dm_file.cpp b/dbms/src/Storages/DeltaMerge/tests/gtest_dm_file.cpp index f186c0d1c2e..13a38b4a1ad 100644 --- a/dbms/src/Storages/DeltaMerge/tests/gtest_dm_file.cpp +++ b/dbms/src/Storages/DeltaMerge/tests/gtest_dm_file.cpp @@ -24,7 +24,7 @@ #include #include #include -#include +#include #include #include #include diff --git a/dbms/src/Storages/DeltaMerge/tests/gtest_dm_segment.cpp b/dbms/src/Storages/DeltaMerge/tests/gtest_dm_segment.cpp index f57d0fc4f01..dc6fa1b822c 100644 --- a/dbms/src/Storages/DeltaMerge/tests/gtest_dm_segment.cpp +++ b/dbms/src/Storages/DeltaMerge/tests/gtest_dm_segment.cpp @@ -20,7 +20,7 @@ #include #include #include -#include +#include #include #include #include diff --git a/dbms/src/Storages/DeltaMerge/tests/gtest_dm_segment_common_handle.cpp b/dbms/src/Storages/DeltaMerge/tests/gtest_dm_segment_common_handle.cpp index 1964f6854dd..d6530f0a9c3 100644 --- a/dbms/src/Storages/DeltaMerge/tests/gtest_dm_segment_common_handle.cpp +++ b/dbms/src/Storages/DeltaMerge/tests/gtest_dm_segment_common_handle.cpp @@ -16,7 +16,7 @@ #include #include #include -#include +#include #include #include #include diff --git a/dbms/src/Storages/DeltaMerge/tests/gtest_dm_segment_s3.cpp b/dbms/src/Storages/DeltaMerge/tests/gtest_dm_segment_s3.cpp index e796629c516..e7e47c0705d 100644 --- a/dbms/src/Storages/DeltaMerge/tests/gtest_dm_segment_s3.cpp +++ b/dbms/src/Storages/DeltaMerge/tests/gtest_dm_segment_s3.cpp @@ -18,7 +18,7 @@ #include #include #include -#include +#include #include #include #include diff --git a/dbms/src/Storages/DeltaMerge/tests/gtest_dm_storage_delta_merge.cpp b/dbms/src/Storages/DeltaMerge/tests/gtest_dm_storage_delta_merge.cpp index e1bd4312d60..756e6025f4e 100644 --- a/dbms/src/Storages/DeltaMerge/tests/gtest_dm_storage_delta_merge.cpp +++ b/dbms/src/Storages/DeltaMerge/tests/gtest_dm_storage_delta_merge.cpp @@ -34,7 +34,7 @@ #include #include #include -#include +#include #include #include #include diff --git a/dbms/src/Storages/DeltaMerge/tests/gtest_segment_test_basic.cpp b/dbms/src/Storages/DeltaMerge/tests/gtest_segment_test_basic.cpp index cade5b28589..387c4af06e0 100644 --- a/dbms/src/Storages/DeltaMerge/tests/gtest_segment_test_basic.cpp +++ b/dbms/src/Storages/DeltaMerge/tests/gtest_segment_test_basic.cpp @@ -19,7 +19,7 @@ #include #include #include -#include +#include #include #include #include diff --git a/dbms/src/Storages/DeltaMerge/tests/gtest_segment_test_basic.h b/dbms/src/Storages/DeltaMerge/tests/gtest_segment_test_basic.h index 931f6d0ba8e..9481ee291b1 100644 --- a/dbms/src/Storages/DeltaMerge/tests/gtest_segment_test_basic.h +++ b/dbms/src/Storages/DeltaMerge/tests/gtest_segment_test_basic.h @@ -17,7 +17,7 @@ #include #include #include -#include +#include #include #include #include diff --git a/dbms/src/Storages/KVStore/MultiRaft/RegionPersister.cpp b/dbms/src/Storages/KVStore/MultiRaft/RegionPersister.cpp index 2e74d757c89..c044b0c8816 100644 --- a/dbms/src/Storages/KVStore/MultiRaft/RegionPersister.cpp +++ b/dbms/src/Storages/KVStore/MultiRaft/RegionPersister.cpp @@ -17,7 +17,7 @@ #include #include #include -#include +#include #include #include #include diff --git a/dbms/src/Storages/Page/V3/Universal/tests/gtest_universal_page_storage_storage_pool.cpp b/dbms/src/Storages/Page/V3/Universal/tests/gtest_universal_page_storage_storage_pool.cpp index cf58331d5f6..ec9a035f19a 100644 --- a/dbms/src/Storages/Page/V3/Universal/tests/gtest_universal_page_storage_storage_pool.cpp +++ b/dbms/src/Storages/Page/V3/Universal/tests/gtest_universal_page_storage_storage_pool.cpp @@ -14,7 +14,7 @@ #include #include -#include +#include #include #include #include diff --git a/dbms/src/Storages/Page/V3/tests/gtest_page_storage_mix_mode.cpp b/dbms/src/Storages/Page/V3/tests/gtest_page_storage_mix_mode.cpp index b3514f036b8..d1bef57238e 100644 --- a/dbms/src/Storages/Page/V3/tests/gtest_page_storage_mix_mode.cpp +++ b/dbms/src/Storages/Page/V3/tests/gtest_page_storage_mix_mode.cpp @@ -14,7 +14,7 @@ #include #include -#include +#include #include #include #include diff --git a/dbms/src/TestUtils/TiFlashTestEnv.cpp b/dbms/src/TestUtils/TiFlashTestEnv.cpp index 77552955d12..d61ddb52edd 100644 --- a/dbms/src/TestUtils/TiFlashTestEnv.cpp +++ b/dbms/src/TestUtils/TiFlashTestEnv.cpp @@ -23,7 +23,7 @@ #include #include #include -#include +#include #include #include #include