From 9309241d874be3c0fb086bd95348cd1e512a0bbf Mon Sep 17 00:00:00 2001 From: CalvinNeo Date: Tue, 4 Jun 2024 18:12:40 +0800 Subject: [PATCH] a Signed-off-by: CalvinNeo --- dbms/src/Common/MemoryAllocTrace.cpp | 33 +++++++++++++++++ dbms/src/Common/MemoryAllocTrace.h | 20 +++++++++++ dbms/src/Common/TiFlashMetrics.cpp | 35 +++++++++++++++++++ dbms/src/Common/TiFlashMetrics.h | 5 +++ .../KVStore/FFI/JointThreadAllocInfo.h | 30 ++++++++++++++++ dbms/src/Storages/KVStore/KVStore.cpp | 2 ++ dbms/src/Storages/KVStore/KVStore.h | 1 + .../KVStore/MultiRaft/Persistence.cpp | 3 ++ 8 files changed, 129 insertions(+) diff --git a/dbms/src/Common/MemoryAllocTrace.cpp b/dbms/src/Common/MemoryAllocTrace.cpp index 35d5ebc67f4..cf4ae4129d3 100644 --- a/dbms/src/Common/MemoryAllocTrace.cpp +++ b/dbms/src/Common/MemoryAllocTrace.cpp @@ -14,6 +14,7 @@ #include #include // Included for `USE_JEMALLOC` +#include #if USE_JEMALLOC #include @@ -35,4 +36,36 @@ std::tuple getAllocDeallocPtr() return std::make_tuple(nullptr, nullptr); #endif } + +ThreadStackAllocTracker::ThreadStackAllocTracker() { + std::tie(alloc_ptr, dealloc_ptr) = getAllocDeallocPtr(); +} + +void ThreadStackAllocTracker::begin() { + begin_alloc = 0; + begin_dealloc = 0; + if likely(alloc_ptr && dealloc_ptr) { + begin_alloc = *alloc_ptr; + begin_dealloc = *dealloc_ptr; + } +} +std::pair ThreadStackAllocTracker::end() { + if likely(alloc_ptr && dealloc_ptr) { + uint64_t delta_alloc = *alloc_ptr - begin_alloc; + uint64_t delta_dealloc = *dealloc_ptr - begin_dealloc; + total_alloc += delta_alloc; + total_dealloc += delta_dealloc; + return std::make_pair(delta_alloc, delta_dealloc); + } + return std::make_pair(0, 0); +} + +std::string ThreadStackAllocTracker::threadName() const { + return getThreadName(); +} + +void ThreadStackAllocTracker::report() { + +} + } // namespace DB diff --git a/dbms/src/Common/MemoryAllocTrace.h b/dbms/src/Common/MemoryAllocTrace.h index fc6303ac3b5..00678505264 100644 --- a/dbms/src/Common/MemoryAllocTrace.h +++ b/dbms/src/Common/MemoryAllocTrace.h @@ -15,8 +15,28 @@ #pragma once #include +#include +#include namespace DB { std::tuple getAllocDeallocPtr(); + +struct ThreadStackAllocTracker { + ThreadStackAllocTracker(); + + void begin(); + std::pair end(); + void report(); + + std::string threadName() const; +private: + uint64_t * alloc_ptr = nullptr; + uint64_t * dealloc_ptr = nullptr; + UInt64 total_alloc = 0; + UInt64 total_dealloc = 0; + UInt64 begin_alloc = 0; + UInt64 begin_dealloc = 0; +}; + } // namespace DB \ No newline at end of file diff --git a/dbms/src/Common/TiFlashMetrics.cpp b/dbms/src/Common/TiFlashMetrics.cpp index f7f47bac33a..710a1ac1cdb 100644 --- a/dbms/src/Common/TiFlashMetrics.cpp +++ b/dbms/src/Common/TiFlashMetrics.cpp @@ -202,4 +202,39 @@ void TiFlashMetrics::registerStorageThreadMemory(const std::string & k) } } +static std::string genStackPrefix(TiFlashMetrics::MemoryAllocType type, const std::string & k) +{ + if (type == TiFlashMetrics::MemoryAllocType::Alloc) + { + return "salloc_" + k; + } + else + { + return "sdealloc_" + k; + } +} + +void TiFlashMetrics::registerStackThreadMemory(const std::string & k) +{ + std::unique_lock lock(stack_thread_report_mtx); + { + auto prefix = genStackPrefix(TiFlashMetrics::MemoryAllocType::Alloc, k); + if unlikely (!registered_stack_thread_memory_usage_metrics.contains(prefix)) + { + registered_stack_thread_memory_usage_metrics.emplace( + prefix, + ®istered_stack_thread_memory_usage_family->Add({{"type", prefix}})); + } + } + { + auto prefix = genStackPrefix(TiFlashMetrics::MemoryAllocType::Dealloc, k); + if unlikely (!registered_stack_thread_memory_usage_metrics.contains(prefix)) + { + registered_stack_thread_memory_usage_metrics.emplace( + prefix, + ®istered_stack_thread_memory_usage_family->Add({{"type", prefix}})); + } + } +} + } // namespace DB \ No newline at end of file diff --git a/dbms/src/Common/TiFlashMetrics.h b/dbms/src/Common/TiFlashMetrics.h index 996c1b0a785..61f2963c821 100644 --- a/dbms/src/Common/TiFlashMetrics.h +++ b/dbms/src/Common/TiFlashMetrics.h @@ -1140,6 +1140,7 @@ class TiFlashMetrics static constexpr auto async_metrics_prefix = "tiflash_system_asynchronous_metric_"; static constexpr auto raft_proxy_thread_memory_usage = "tiflash_raft_proxy_thread_memory_usage"; static constexpr auto storages_thread_memory_usage = "tiflash_storages_thread_memory_usage"; + static constexpr auto stack_thread_memory_usage = "tiflash_stack_thread_memory_usage"; std::shared_ptr registry = std::make_shared(); // Here we add a ProcessCollector to collect cpu/rss/vsize/start_time information. @@ -1169,6 +1170,10 @@ class TiFlashMetrics std::shared_mutex storage_thread_report_mtx; std::unordered_map registered_storage_thread_memory_usage_metrics; + prometheus::Family * registered_stack_thread_memory_usage_family; + std::shared_mutex stack_thread_report_mtx; + std::unordered_map registered_stack_thread_memory_usage_metrics; + public: #define MAKE_METRIC_MEMBER_M(family_name, help, type, ...) \ MetricFamily family_name \ diff --git a/dbms/src/Storages/KVStore/FFI/JointThreadAllocInfo.h b/dbms/src/Storages/KVStore/FFI/JointThreadAllocInfo.h index 7d4a500a4e2..77554a33776 100644 --- a/dbms/src/Storages/KVStore/FFI/JointThreadAllocInfo.h +++ b/dbms/src/Storages/KVStore/FFI/JointThreadAllocInfo.h @@ -67,6 +67,26 @@ struct ThreadInfoJealloc } }; +struct StackThreadInfoJealloc { + explicit StackThreadInfoJealloc(char aggregate_delimer_) + : aggregate_delimer(aggregate_delimer_) + {} + char aggregate_delimer = '-'; + uint64_t allocated{0}; + uint64_t deallocated{0}; + int64_t remaining() const + { + if (allocated > deallocated) + { + return static_cast(allocated - deallocated); + } + else + { + return -static_cast(deallocated - allocated); + } + } +}; + /// Works in two different ways: /// NOTE in both ways, call reportThreadAllocInfo to register by `Reset` for every thread to be monitored. /// And call reportThreadAllocInfo to deregister by `Remove` for every thread that is guaranteed to no longer be monitored. @@ -79,6 +99,7 @@ class JointThreadInfoJeallocMap { public: using AllocMap = std::unordered_map; + using StackAllocMap = std::unordered_map; JointThreadInfoJeallocMap(); ~JointThreadInfoJeallocMap(); void recordThreadAllocInfo(); @@ -106,11 +127,18 @@ class JointThreadInfoJeallocMap uint64_t value, char aggregate_delimer); +public: // StackThread + void reportStackThreadAllocInfo( + const std::string &, + StackThreadInfoJealloc, + char aggregate_delimer); + private: /// Be called periodicly to submit the alloc info to TiFlashMetrics /// Note that this function rely on `TiFlashMetrics::instance` is alive void recordThreadAllocInfoForProxy(); void recordThreadAllocInfoForStorage(); + void recordStackThreadAllocInfo(); /// Note that this function rely on `TiFlashMetrics::instance` is alive void reportThreadAllocInfoImpl( @@ -122,8 +150,10 @@ class JointThreadInfoJeallocMap private: mutable std::shared_mutex memory_allocation_mut; + mutable std::shared_mutex stack_memory_allocation_mut; AllocMap proxy_map; AllocMap storage_map; + StackAllocMap stack_map; bool is_terminated{false}; mutable std::mutex monitoring_mut; diff --git a/dbms/src/Storages/KVStore/KVStore.cpp b/dbms/src/Storages/KVStore/KVStore.cpp index f2c811bae0a..42630ee32df 100644 --- a/dbms/src/Storages/KVStore/KVStore.cpp +++ b/dbms/src/Storages/KVStore/KVStore.cpp @@ -53,6 +53,8 @@ extern const char pause_passive_flush_before_persist_region[]; extern const char force_not_clean_fap_on_destroy[]; } // namespace FailPoints +thread_local ThreadStackAllocTracker mem_tracker_kvs_memtable_persistence; + KVStore::KVStore(Context & context) : region_persister( context.getSharedContextDisagg()->isDisaggregatedComputeMode() ? nullptr diff --git a/dbms/src/Storages/KVStore/KVStore.h b/dbms/src/Storages/KVStore/KVStore.h index d70e09d02c0..f2ee51d3e29 100644 --- a/dbms/src/Storages/KVStore/KVStore.h +++ b/dbms/src/Storages/KVStore/KVStore.h @@ -24,6 +24,7 @@ #include #include #include +#include #include #include diff --git a/dbms/src/Storages/KVStore/MultiRaft/Persistence.cpp b/dbms/src/Storages/KVStore/MultiRaft/Persistence.cpp index b34d863d88e..12ea48cc81b 100644 --- a/dbms/src/Storages/KVStore/MultiRaft/Persistence.cpp +++ b/dbms/src/Storages/KVStore/MultiRaft/Persistence.cpp @@ -39,6 +39,9 @@ void KVStore::persistRegion( "try access to region_persister without initialization, stack={}", StackTrace().toString()); + mem_tracker_kvs_memtable_persistence.begin(); + SCOPE_EXIT({ mem_tracker_kvs_memtable_persistence.end(); }); + auto reason_id = magic_enum::enum_underlying(reason); std::string caller = fmt::format("{} {}", PersistRegionReasonMap[reason_id], extra_msg); LOG_INFO(