[#17727] docdb: Use allocated_size instead of size / count in tcmallo…

…c sampling report.

Summary:
The sampling profiler introduced as part of Google TCMalloc outputs profiles of sampled objects.

The existing code aggregates samples by the sample object's stack, and adds the `sum` and `count` fields for samples with the same stack. The `count` is based on how many allocations a sample period could contain. For example, a 100 KB allocation would have a `count` of 10 if the sample frequency is 1 MB, since the full 1 MB could be made up of 10 100 KB allocations. `sum` is `count` multiplied by the allocated size (1 MB = 10 * 100 KB).

This is confusing, since we might not actually be allocating `count` objects. It's much simpler to emit the `allocated_size` of the object we actually sampled.
Jira: DB-6826

Test Plan:
`ybd --cxx-test pprof-path-handler_util-test --gtest_filter SamplingProfilerTest.HeapSnapshot -n 1000`
`ybd --cxx-test pprof-path-handler_util-test --gtest_filter SamplingProfilerTest.AllocationProfile -n 1000`
Jenkins: run all tests

Reviewers: mbautin

Reviewed By: mbautin

Subscribers: bogdan, ybase

Differential Revision: https://phorge.dev.yugabyte.com/D26126

src/yb/server/CMakeLists.txt

@@ -129,3 +129,4 @@ endif()
 
 set(YB_TEST_LINK_LIBS server_process ${YB_MIN_TEST_LIBS})
 ADD_YB_TEST(webserver-test)
+ADD_YB_TEST(pprof-path-handler_util-test)

src/yb/server/pprof-path-handler_util-test.cc

@@ -0,0 +1,125 @@
+// Copyright (c) YugabyteDB, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
+// in compliance with the License.  You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software distributed under the License
+// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
+// or implied.  See the License for the specific language governing permissions and limitations
+// under the License.
+
+#include <memory>
+#include <string>
+#include <thread>
+
+#include <gflags/gflags_declare.h>
+#include <glog/logging.h>
+#include <gtest/gtest.h>
+
+#include "yb/gutil/dynamic_annotations.h"
+#include "yb/server/pprof-path-handlers_util.h"
+
+#include "yb/util/flags.h"
+#include "yb/util/monotime.h"
+#include "yb/util/size_literals.h"
+#include "yb/util/test_util.h"
+
+DECLARE_int32(v);
+DECLARE_string(vmodule);
+
+namespace yb {
+
+#if YB_GOOGLE_TCMALLOC
+
+class SamplingProfilerTest : public YBTest {
+ public:
+  void SetUp() override {
+    YBTest::SetUp();
+    ASSERT_OK(EnableVerboseLoggingForModule("pprof-path-handlers_util", 2));
+  }
+};
+
+std::unique_ptr<char[]> AllocArrayOfSize(int64_t alloc_size) {
+  std::unique_ptr<char[]> alloc(new char[alloc_size]);
+  // Clang in release mode can optimize out the above allocation unless
+  // we do something with the pointer... so we just log it.
+  VLOG(8) << static_cast<void*>(alloc.get());
+  return alloc;
+}
+
+std::vector<Sample> GetStacksFromHeapSnapshot(HeapSnapshotType snapshot_type) {
+  auto current_profile = GetHeapSnapshot(snapshot_type);
+  return AggregateAndSortProfile(current_profile, false /* only_growth */);
+}
+
+int GetNumAllocsOfSizeAtLeast(int64_t size, const std::vector<Sample>& stacks) {
+  int val = 0;
+  for (const auto& stack : stacks) {
+    if (stack.second.bytes >= size) {
+      ++val;
+    }
+  }
+  return val;
+}
+
+TEST_F(SamplingProfilerTest, HeapSnapshot) {
+  // Since TCMalloc's bytes_until_sample is set upon initialization to approximately 2 MB,
+  // (based on an exponential distribution with mean 2 MB), setting the sampling rate below only
+  // affects the samples after the bytes_until_sample bytes are allocated.
+  tcmalloc::MallocExtension::SetProfileSamplingRate(1);
+  // 30 MB will be sampled with probablility > 99.9999%.
+  const int64_t alloc_size = 30_MB;
+  {
+    // Make a large allocation. We expect to find it in the current and peak heap snapshots.
+    std::unique_ptr<char[]> big_alloc = AllocArrayOfSize(alloc_size);
+
+    auto stacks = GetStacksFromHeapSnapshot(HeapSnapshotType::CURRENT_HEAP);
+    ASSERT_EQ(GetNumAllocsOfSizeAtLeast(alloc_size, stacks), 1);
+
+    stacks = GetStacksFromHeapSnapshot(HeapSnapshotType::PEAK_HEAP);
+    ASSERT_EQ(GetNumAllocsOfSizeAtLeast(alloc_size, stacks), 1);
+  }
+  // After the deallocation, the stack should no longer be found in the current heap snapshot,
+  // but should be in the peak heap snapshot.
+  auto stacks = GetStacksFromHeapSnapshot(HeapSnapshotType::CURRENT_HEAP);
+  ASSERT_EQ(GetNumAllocsOfSizeAtLeast(alloc_size, stacks), 0);
+
+  stacks = GetStacksFromHeapSnapshot(HeapSnapshotType::PEAK_HEAP);
+  ASSERT_EQ(GetNumAllocsOfSizeAtLeast(alloc_size, stacks), 1);
+}
+
+TEST_F(SamplingProfilerTest, AllocationProfile) {
+  tcmalloc::MallocExtension::SetProfileSamplingRate(1);
+  // 30 MB will be sampled with probablility > 99.9999%.
+  const int64_t alloc_size = 30_MB;
+
+  tcmalloc::MallocExtension::AllocationProfilingToken token;
+  token = tcmalloc::MallocExtension::StartLifetimeProfiling();
+
+  // We expect to find this allocation in the profile if and only if only_growth is false, since
+  // it is not deallocated before we stop profiling.
+  std::unique_ptr<char[]> big_alloc = AllocArrayOfSize(alloc_size);
+
+  // We expect to always find this allocation in the profile since it is deallocated before we stop
+  // profiling.
+  std::unique_ptr<char[]> big_alloc2 = AllocArrayOfSize(alloc_size);
+  big_alloc2.reset();
+
+  auto profile = std::move(token).Stop();
+
+  // The stack for both allocations is the same so they are aggregated into one.
+  auto stacks = AggregateAndSortProfile(profile, false /* only_growth */);
+  ASSERT_EQ(GetNumAllocsOfSizeAtLeast(alloc_size, stacks), 1);
+  ASSERT_GE(stacks[0].second.bytes, alloc_size * 2);
+
+  // We only expect to find the non-deallocated allocation here.
+  stacks = AggregateAndSortProfile(profile, true /* only_growth */);
+  ASSERT_EQ(GetNumAllocsOfSizeAtLeast(alloc_size, stacks), 1);
+  ASSERT_GE(stacks[0].second.bytes, alloc_size);
+}
+
+#endif // YB_GOOGLE_TCMALLOC
+
+} // namespace yb

src/yb/server/pprof-path-handlers.cc

@@ -176,7 +176,8 @@ static void PprofHeapSnapshotHandler(const Webserver::WebRequest& req,
 
   bool peak_heap = ParseLeadingBoolValue(FindWithDefault(req.parsed_args, "peak_heap", ""), false);
   string title = peak_heap ? "Peak heap snapshot" : "Current heap snapshot";
-  auto profile = GetHeapSnapshot(peak_heap);
+  auto profile = peak_heap ? GetHeapSnapshot(HeapSnapshotType::PEAK_HEAP) :
+                             GetHeapSnapshot(HeapSnapshotType::CURRENT_HEAP);
   auto samples = AggregateAndSortProfile(profile, false /* only_growth */);
   GenerateTable(output, samples, title, 1000 /* max_call_stacks */);
 #else

src/yb/server/pprof-path-handlers_util.cc

@@ -10,37 +10,40 @@
 // or implied.  See the License for the specific language governing permissions and limitations
 // under the License.
 
+#if YB_GOOGLE_TCMALLOC
+
 #include "yb/server/pprof-path-handlers_util.h"
 
-#include <glog/logging.h>
-#if YB_GOOGLE_TCMALLOC
 #include <tcmalloc/malloc_extension.h>
-#endif
 
 #include <cstdint>
 #include <iomanip>
 #include <string>
 #include <utility>
 
+#include <glog/logging.h>
+
 #include "yb/util/format.h"
 #include "yb/util/monotime.h"
 
 
 DECLARE_bool(enable_process_lifetime_heap_profiling);
 
-// GLog already implements symbolization. Just import their hidden symbol.
-namespace google {
-// Symbolizes a program counter.  On success, returns true and write the
-// symbol name to "out".  The symbol name is demangled if possible
-// (supports symbols generated by GCC 3.x or newer).  Otherwise,
-// returns false.
-bool Symbolize(void *pc, char *out, int out_size);
+// Abseil already implements symbolization. Just import their hidden symbol.
+namespace absl {
+
+// Symbolizes a program counter (instruction pointer value) `pc` and, on
+// success, writes the name to `out`. The symbol name is demangled, if possible.
+// Note that the symbolized name may be truncated and will be NUL-terminated.
+// Demangling is supported for symbols generated by GCC 3.x or newer). Returns
+// `false` on failure.
+bool Symbolize(const void *pc, char *out, int out_size);
+
 }
 
 
 namespace yb {
 
-#if YB_GOOGLE_TCMALLOC
 
 tcmalloc::Profile GetAllocationProfile(int seconds, int64_t sample_freq_bytes) {
   auto prev_sample_rate = tcmalloc::MallocExtension::GetProfileSamplingRate();
@@ -54,42 +57,59 @@ tcmalloc::Profile GetAllocationProfile(int seconds, int64_t sample_freq_bytes) {
   return std::move(token).Stop();
 }
 
-// If peak_heap is set, gets the snapshot of the heap at peak memory usage.
-tcmalloc::Profile GetHeapSnapshot(bool peak_heap) {
-  auto profile_type = peak_heap ? tcmalloc::ProfileType::kPeakHeap : tcmalloc::ProfileType::kHeap;
-  return tcmalloc::MallocExtension::SnapshotCurrent(profile_type);
+tcmalloc::Profile GetHeapSnapshot(HeapSnapshotType snapshot_type) {
+  if (snapshot_type == PEAK_HEAP) {
+    return tcmalloc::MallocExtension::SnapshotCurrent(tcmalloc::ProfileType::kPeakHeap);
+  } else {
+    return tcmalloc::MallocExtension::SnapshotCurrent(tcmalloc::ProfileType::kHeap);
+  }
 }
 
 std::vector<Sample> AggregateAndSortProfile(const tcmalloc::Profile& profile, bool only_growth) {
-  LOG(INFO) << "Analyzing profile";
+  LOG(INFO) << "Analyzing TCMalloc sampling profile";
   int failed_symbolizations = 0;
   std::unordered_map<std::string, SampleInfo> samples_map;
 
   profile.Iterate([&](const tcmalloc::Profile::Sample& sample) {
-    // Ignore deallocations.
+    // Deallocation samples are the same as the allocation samples, except with a negative
+    // sample.count < 0 and the deallocation stack. Skip since we are not currently interested in
+    // printing the deallocation stack.
     if (sample.count <= 0) {
       return;
     }
+
     // If we only want growth, exclude samples for which we saw a deallocation event.
+    // "Censored" means we observed an allocation but not a deallocation. (Deallocation-only events
+    // are not reported).
     if (only_growth && !sample.is_censored) {
       return;
     }
+
     std::stringstream sstream;
     char buf[256];
     for (int64_t i = 0; i < sample.depth; ++i) {
-      if (!google::Symbolize(sample.stack[i], buf, sizeof(buf))) {
-        sstream << "<failed to symbolize>";
+      if (absl::Symbolize(sample.stack[i], buf, sizeof(buf))) {
+        sstream << buf << std::endl;
+      } else {
         ++failed_symbolizations;
-      }
-      sstream << buf;
-      if (i != sample.depth) {
-        sstream << "\n";
+        sstream << "Failed to symbolize" << std::endl;
       }
     }
     std::string stack = sstream.str();
+
     auto& entry = samples_map[stack];
-    entry.bytes += sample.sum;
-    entry.count += sample.count;
+    entry.bytes += sample.allocated_size;
+    ++entry.count;
+
+    VLOG(1) << "Sampled stack: " << stack
+            << ", sum: " << sample.sum
+            << ", count: " << sample.count
+            << ", requested_size: " << sample.requested_size
+            << ", allocated_size: " << sample.allocated_size
+            << ", is_censored: " << sample.is_censored
+            << ", avg_lifetime: " << sample.avg_lifetime
+            << ", allocator_deallocator_cpu_matched: "
+            << sample.allocator_deallocator_cpu_matched.value_or("N/A");
   });
   if (failed_symbolizations > 0) {
     LOG(WARNING) << Format("Failed to symbolize $0 symbols", failed_symbolizations);
@@ -136,6 +156,6 @@ void GenerateTable(std::stringstream* output, const std::vector<Sample>& samples
   (*output) << "</table>";
 }
 
-#endif // YB_GOOGLE_TCMALLOC
-
 } // namespace yb
+
+#endif // YB_GOOGLE_TCMALLOC

src/yb/server/pprof-path-handlers_util.h

@@ -13,8 +13,8 @@
 #pragma once
 
 #if YB_GOOGLE_TCMALLOC
+
 #include <tcmalloc/malloc_extension.h>
-#endif // YB_GOOGLE_TCMALLOC
 
 #include <cstdint>
 #include <string>
@@ -28,8 +28,6 @@ DECLARE_bool(enable_process_lifetime_heap_profiling);
 
 namespace yb {
 
-#if YB_GOOGLE_TCMALLOC
-
 struct SampleInfo {
   int64_t bytes;
   int64_t count;
@@ -39,14 +37,19 @@ typedef std::pair<std::string, SampleInfo> Sample;
 
 tcmalloc::Profile GetAllocationProfile(int seconds, int64_t sample_freq_bytes);
 
+enum HeapSnapshotType {
+  CURRENT_HEAP,
+  PEAK_HEAP
+};
+
 // If peak_heap is set, gets the snapshot of the heap at peak memory usage.
-tcmalloc::Profile GetHeapSnapshot(bool peak_heap);
+tcmalloc::Profile GetHeapSnapshot(HeapSnapshotType snapshot_type);
 
 std::vector<Sample> AggregateAndSortProfile(const tcmalloc::Profile& profile, bool only_growth);
 
 void GenerateTable(std::stringstream* output, const std::vector<Sample>& samples,
     const std::string& title, size_t max_call_stacks);
 
-#endif // YB_GOOGLE_TCMALLOC
-
 } // namespace yb
+
+#endif // YB_GOOGLE_TCMALLOC

src/yb/tserver/server_main_util.cc

@@ -30,6 +30,14 @@
 
 DECLARE_int64(remote_bootstrap_rate_limit_bytes_per_sec);
 
+// Import the internal symbol for the Abseil symbolizer (required for pprof endpoints when
+// YB_GOOGLE_TCMALLOC is enabled).
+#if YB_GOOGLE_TCMALLOC
+namespace absl {
+void InitializeSymbolizer(const char* argv0);
+}
+#endif
+
 namespace yb {
 
 Status MasterTServerParseFlagsAndInit(const std::string& server_type, int* argc, char*** argv) {
@@ -50,6 +58,10 @@ Status MasterTServerParseFlagsAndInit(const std::string& server_type, int* argc,
     return STATUS(InvalidArgument, "Error parsing command-line flags");
   }
 
+#if YB_GOOGLE_TCMALLOC
+  absl::InitializeSymbolizer((*argv)[0]);
+#endif
+
   RETURN_NOT_OK(log::ModifyDurableWriteFlagIfNotODirect());
 
   RETURN_NOT_OK(InitYB(server_type, (*argv)[0]));

yb_build.sh

@@ -1047,6 +1047,7 @@ while [[ $# -gt 0 ]]; do
     ;;
     --no-tcmalloc)
       no_tcmalloc=true
+      use_google_tcmalloc=false
     ;;
     --no-google-tcmalloc|--use-gperftools-tcmalloc|--gperftools-tcmalloc)
       use_google_tcmalloc=false