chore: allow measurement of fiber stack utilization

Signed-off-by: Roman Gershman <romange@gmail.com>

util/fibers/CMakeLists.txt

@@ -14,6 +14,11 @@ add_library(fibers2 fibers.cc proactor_base.cc synchronization.cc
             sliding_counter.cc varz.cc fiberqueue_threadpool.cc dns_resolve.cc
             ${FB_LINUX_SRCS})
 
+option(HELIO_ENABLE_STACK_MEASURE "Enable stack size measuring" OFF)
+if (HELIO_ENABLE_STACK_MEASURE)
+  target_compile_definitions(fibers2 PUBLIC -DCHECK_FIBER_STACK_SIZE_USAGE)
+endif()
+
 cxx_link(fibers2 base io ${FB_LINUX_LIBS} Boost::context Boost::headers TRDP::cares)
 
 cxx_test(fibers_test fibers2 LABELS CI)

util/fibers/detail/fiber_interface.cc

@@ -384,6 +384,16 @@ void ExecuteOnAllFiberStacks(FiberInterface::PrintFn fn) {
   FbInitializer().sched->ExecuteOnAllFiberStacks(std::move(fn));
 }
 
+
+void PrintFiberStackMargin(const void* bottom, const char* name) {
+  uint8_t* stack_bottom = (uint8_t*)bottom;
+  uint8_t* ptr = stack_bottom;
+  while (*ptr == 0xAB) {
+    ++ptr;
+  }
+  VLOG(1) << "Stack margin for " << name << ": " << (ptr - stack_bottom) << " bytes";
+}
+
 }  // namespace detail
 
 void SetCustomDispatcher(DispatchPolicy* policy) {

util/fibers/detail/fiber_interface.h

@@ -59,6 +59,11 @@ using FI_SleepHook =
 
 class Scheduler;
 
+// Enable this define via cmake option
+#ifdef CHECK_FIBER_STACK_SIZE_USAGE
+  void PrintFiberStackMargin(const void* bottom, const char* name);
+#endif
+
 class FiberInterface {
   friend class Scheduler;
 
@@ -269,7 +274,13 @@ template <typename Fn, typename... Arg> class WorkerFiberImpl : public FiberInte
                   StackAlloc&& salloc, Fn&& fn, Arg&&... arg)
       : FiberInterface(WORKER, 1, name), fn_(std::forward<Fn>(fn)),
         arg_(std::forward<Arg>(arg)...) {
-    stack_size_ = palloc.sctx.size;
+    stack_size_ = palloc.sctx.size;  // The whole stack size that was allocated for this fiber.
+
+#ifdef CHECK_FIBER_STACK_SIZE_USAGE
+      stack_bottom_ = reinterpret_cast<uint8_t*>(palloc.sp) - palloc.size;
+      memset(stack_bottom_, 0xAB, palloc.size);
+#endif
+
     entry_ = FbCntx(std::allocator_arg, palloc, std::forward<StackAlloc>(salloc),
                     [this](FbCntx&& caller) { return run_(std::move(caller)); });
 #if defined(BOOST_USE_UCONTEXT)
@@ -292,13 +303,19 @@ template <typename Fn, typename... Arg> class WorkerFiberImpl : public FiberInte
 
       std::apply(std::move(fn), std::move(arg));
     }
-
+#ifdef CHECK_FIBER_STACK_SIZE_USAGE
+    PrintFiberStackMargin(stack_bottom_, name_);
+#endif
     return Terminate();
   }
 
   // Without decay - fn_ can be a reference, depending how a function is passed to the constructor.
   typename std::decay<Fn>::type fn_;
   std::tuple<std::decay_t<Arg>...> arg_;
+
+#ifdef CHECK_FIBER_STACK_SIZE_USAGE
+  void* stack_bottom_;
+#endif
 };
 
 template <typename FbImpl>
@@ -308,8 +325,8 @@ boost::context::preallocated MakePreallocated(const boost::context::stack_contex
   uintptr_t fb_impl_ptr =
       (reinterpret_cast<uintptr_t>(sctx.sp) - sizeof(FbImpl)) & ~static_cast<uintptr_t>(0xff);
 
-  // stack_bottom is the real pointer allocated inside salloc.allocate()
-  // sctx.sp, on the other hand, points to the top (right boundary) of the allocated region.
+  // stack_bottom is the real pointer allocated by salloc.allocate()
+  // sctx.sp, points to the top (right boundary) of the allocated region.
   // The deeper the stack grows, the closer it gets to the stack_bottom.
   uintptr_t stack_bottom = reinterpret_cast<uintptr_t>(sctx.sp) - static_cast<uintptr_t>(sctx.size);
   const std::size_t size = fb_impl_ptr - stack_bottom;  // effective stack size.
@@ -329,7 +346,8 @@ static WorkerFiberImpl<Fn, Arg...>* MakeWorkerFiberImpl(std::string_view name, S
 
   void* obj_ptr = palloc.sp;  // copy because we move palloc.
 
-  // placement new of context on top of fiber's stack
+  // placement new of context on top of fiber's stack.
+  // note, that obj_ptr is not real top of the stack, because boost places more records below it.
   WorkerImpl* fctx =
       new (obj_ptr) WorkerImpl{name, std::move(palloc), std::forward<StackAlloc>(salloc),
                                std::forward<Fn>(fn), std::forward<Arg>(arg)...};