chore: allow measurement of fiber stack utilization (#346)

Signed-off-by: Roman Gershman <romange@gmail.com>

util/aws/http_client.cc

@@ -256,7 +256,7 @@ io::Result<boost::asio::ip::address> HttpClient::Resolve(const std::string& host
   VLOG(1) << "aws: http client: resolving host; host=" << host;
 
   char ip[INET6_ADDRSTRLEN];
-  std::error_code ec = fb2::DnsResolve(host.data(), client_conf_.connectTimeoutMs, ip, proactor);
+  std::error_code ec = fb2::DnsResolve(host, client_conf_.connectTimeoutMs, ip, proactor);
   if (ec) {
     LOG(WARNING) << "aws: http client: failed to resolve host; host=" << host << "; error=" << ec;
     return nonstd::make_unexpected(ec);

util/fibers/CMakeLists.txt

@@ -14,6 +14,11 @@ add_library(fibers2 fibers.cc proactor_base.cc synchronization.cc
             sliding_counter.cc varz.cc fiberqueue_threadpool.cc dns_resolve.cc
             ${FB_LINUX_SRCS})
 
+option(HELIO_ENABLE_STACK_MEASURE "Enable stack size measuring" OFF)
+if (HELIO_ENABLE_STACK_MEASURE)
+  target_compile_definitions(fibers2 PUBLIC -DCHECK_FIBER_STACK_SIZE_USAGE)
+endif()
+
 cxx_link(fibers2 base io ${FB_LINUX_LIBS} Boost::context Boost::headers TRDP::cares)
 
 cxx_test(fibers_test fibers2 LABELS CI)

util/fibers/detail/fiber_interface.cc

@@ -384,6 +384,23 @@ void ExecuteOnAllFiberStacks(FiberInterface::PrintFn fn) {
   FbInitializer().sched->ExecuteOnAllFiberStacks(std::move(fn));
 }
 
+
+void PrintFiberStackMargin(const void* bottom, const char* name) {
+  uint8_t* stack_bottom = (uint8_t*)bottom;
+  uint8_t* ptr = stack_bottom;
+  while (*ptr == 0xAB) {
+    ++ptr;
+  }
+  VLOG(1) << "Stack margin for " << name << ": " << (ptr - stack_bottom) << " bytes";
+}
+
+void ActivateSameThread(FiberInterface* active, FiberInterface* other) {
+  if (active->scheduler() != other->scheduler()) {
+    LOG(DFATAL) << "Fibers belong to different schedulers";
+  }
+  active->ActivateOther(other);
+}
+
 }  // namespace detail
 
 void SetCustomDispatcher(DispatchPolicy* policy) {

util/fibers/detail/fiber_interface.h

@@ -59,6 +59,11 @@ using FI_SleepHook =
 
 class Scheduler;
 
+// Enable this define via cmake option
+#ifdef CHECK_FIBER_STACK_SIZE_USAGE
+  void PrintFiberStackMargin(const void* bottom, const char* name);
+#endif
+
 class FiberInterface {
   friend class Scheduler;
 
@@ -269,7 +274,13 @@ template <typename Fn, typename... Arg> class WorkerFiberImpl : public FiberInte
                   StackAlloc&& salloc, Fn&& fn, Arg&&... arg)
       : FiberInterface(WORKER, 1, name), fn_(std::forward<Fn>(fn)),
         arg_(std::forward<Arg>(arg)...) {
-    stack_size_ = palloc.sctx.size;
+    stack_size_ = palloc.sctx.size;  // The whole stack size that was allocated for this fiber.
+
+#ifdef CHECK_FIBER_STACK_SIZE_USAGE
+      stack_bottom_ = reinterpret_cast<uint8_t*>(palloc.sp) - palloc.size;
+      memset(stack_bottom_, 0xAB, palloc.size);
+#endif
+
     entry_ = FbCntx(std::allocator_arg, palloc, std::forward<StackAlloc>(salloc),
                     [this](FbCntx&& caller) { return run_(std::move(caller)); });
 #if defined(BOOST_USE_UCONTEXT)
@@ -292,13 +303,19 @@ template <typename Fn, typename... Arg> class WorkerFiberImpl : public FiberInte
 
       std::apply(std::move(fn), std::move(arg));
     }
-
+#ifdef CHECK_FIBER_STACK_SIZE_USAGE
+    PrintFiberStackMargin(stack_bottom_, name_);
+#endif
     return Terminate();
   }
 
   // Without decay - fn_ can be a reference, depending how a function is passed to the constructor.
   typename std::decay<Fn>::type fn_;
   std::tuple<std::decay_t<Arg>...> arg_;
+
+#ifdef CHECK_FIBER_STACK_SIZE_USAGE
+  void* stack_bottom_;
+#endif
 };
 
 template <typename FbImpl>
@@ -308,8 +325,8 @@ boost::context::preallocated MakePreallocated(const boost::context::stack_contex
   uintptr_t fb_impl_ptr =
       (reinterpret_cast<uintptr_t>(sctx.sp) - sizeof(FbImpl)) & ~static_cast<uintptr_t>(0xff);
 
-  // stack_bottom is the real pointer allocated inside salloc.allocate()
-  // sctx.sp, on the other hand, points to the top (right boundary) of the allocated region.
+  // stack_bottom is the real pointer allocated by salloc.allocate()
+  // sctx.sp, points to the top (right boundary) of the allocated region.
   // The deeper the stack grows, the closer it gets to the stack_bottom.
   uintptr_t stack_bottom = reinterpret_cast<uintptr_t>(sctx.sp) - static_cast<uintptr_t>(sctx.size);
   const std::size_t size = fb_impl_ptr - stack_bottom;  // effective stack size.
@@ -329,7 +346,8 @@ static WorkerFiberImpl<Fn, Arg...>* MakeWorkerFiberImpl(std::string_view name, S
 
   void* obj_ptr = palloc.sp;  // copy because we move palloc.
 
-  // placement new of context on top of fiber's stack
+  // placement new of context on top of fiber's stack.
+  // note, that obj_ptr is not real top of the stack, because boost places more records below it.
   WorkerImpl* fctx =
       new (obj_ptr) WorkerImpl{name, std::move(palloc), std::forward<StackAlloc>(salloc),
                                std::forward<Fn>(fn), std::forward<Arg>(arg)...};
@@ -347,10 +365,7 @@ void PrintAllFiberStackTraces();
 void ExecuteOnAllFiberStacks(FiberInterface::PrintFn fn);
 
 // A convenience function to improve the readability of the code.
-inline void ActivateSameThread(FiberInterface* active, FiberInterface* other) {
-  assert(active->scheduler() == other->scheduler());
-  active->ActivateOther(other);
-}
+void ActivateSameThread(FiberInterface* active, FiberInterface* other);
 
 extern PMR_NS::memory_resource* default_stack_resource;
 extern size_t default_stack_size;

util/fibers/dns_resolve.cc

@@ -159,7 +159,8 @@ void ProcessChannel(ares_channel channel, AresChannelState* state, DnsResolveCal
 
 }  // namespace
 
-error_code DnsResolve(string host, uint32_t wait_ms, char dest_ip[], ProactorBase* proactor) {
+error_code DnsResolve(const string& host, uint32_t wait_ms, char dest_ip[],
+                      ProactorBase* proactor) {
   DCHECK(ProactorBase::me() == proactor) << "must call from the proactor thread";
   VLOG(1) << "DnsResolveStart";
 
@@ -176,7 +177,6 @@ error_code DnsResolve(string host, uint32_t wait_ms, char dest_ip[], ProactorBas
   DnsResolveCallbackArgs cb_args;
   cb_args.dest_ip = dest_ip;
 
-
   // Same hints as for  hostentares_gethostbyname
   ares_addrinfo_hints hints{ARES_AI_CANONNAME, AF_UNSPEC, 0, 0};
   ares_getaddrinfo(channel, host.c_str(), nullptr, &hints, &DnsResolveCallback, &cb_args);

util/fibers/dns_resolve.h

@@ -13,7 +13,7 @@ namespace fb2 {
 
 class ProactorBase;
 // Resolve addresss - either ipv4 or ipv6. dest_ip must be able to hold INET6_ADDRSTRLEN
-std::error_code DnsResolve(std::string host, uint32_t wait_ms, char dest_ip[],
+std::error_code DnsResolve(const std::string& host, uint32_t wait_ms, char dest_ip[],
                            ProactorBase* proactor);
 
 }  // namespace fb2

util/http/http_client.cc

@@ -71,7 +71,7 @@ std::error_code Client::Reconnect() {
   }
 
   char ip[INET6_ADDRSTRLEN];
-  error_code ec = fb2::DnsResolve(host_.data(), 2000, ip, proactor_);
+  error_code ec = fb2::DnsResolve(host_, 2000, ip, proactor_);
   if (ec) {
     return ec;
   }