Improve crashlog

CMakeLists.txt

@@ -203,10 +203,6 @@ include_directories(
   ${CMAKE_CURRENT_BINARY_DIR} ${CMAKE_CURRENT_BINARY_DIR}/lib
 )
 
-if(HAVE_LIBEXECINFO)
-  list(APPEND base_DEPS execinfo)
-endif()
-
 if(UNIX OR CYGWIN)
   list(APPEND base_OBJS $<TARGET_OBJECTS:execvpe>)
 endif()
@@ -365,6 +361,20 @@ check_include_file_cxx(cxxabi.h HAVE_CXXABI_H)
 
 if(HAVE_LIBEXECINFO)
   set(HAVE_BACKTRACE_SYMBOLS TRUE)
+  list(APPEND base_DEPS execinfo)
+endif()
+
+if(NOT WIN32)
+  # boost::stacktrace uses _Unwind_Backtrace which is only exposed if _GNU_SOURCE is defined on most systems
+  add_definitions(-D_GNU_SOURCE)
+endif()
+
+if(CMAKE_SYSTEM_NAME STREQUAL "FreeBSD")
+  set(ICINGA2_STACKTRACE_USE_BACKTRACE_SYMBOLS TRUE)
+endif()
+
+if(ICINGA2_STACKTRACE_USE_BACKTRACE_SYMBOLS AND NOT HAVE_BACKTRACE_SYMBOLS)
+  message(FATAL_ERROR "ICINGA2_STACKTRACE_USE_BACKTRACE_SYMBOLS is set but backtrace_symbols() was not found")
 endif()
 
 if(CMAKE_CXX_COMPILER_ID STREQUAL "GNU")
@@ -378,6 +388,12 @@ if(CMAKE_CXX_COMPILER_ID STREQUAL "GNU")
   endif()
 endif()
 
+if(MSVC)
+  if("${CMAKE_CXX_COMPILER_VERSION}" VERSION_LESS "19.20")
+    message(FATAL_ERROR "Your version of MSVC (${CMAKE_CXX_COMPILER_VERSION}) is too old for building Icinga 2 (MSVC >= 19.20 from Visual Studio 2019 is required).")
+  endif()
+endif()
+
 if(NOT MSVC)
   set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++0x")
 

config.h.cmake

@@ -12,6 +12,7 @@
 #cmakedefine HAVE_SYSTEMD
 
 #cmakedefine ICINGA2_UNITY_BUILD
+#cmakedefine ICINGA2_STACKTRACE_USE_BACKTRACE_SYMBOLS
 
 #define ICINGA_CONFIGDIR "${ICINGA2_FULL_CONFIGDIR}"
 #define ICINGA_DATADIR "${ICINGA2_FULL_DATADIR}"

doc/21-development.md

@@ -1151,6 +1151,7 @@ General:
 - [function_types](https://www.boost.org/doc/libs/1_66_0/libs/function_types/doc/html/index.html) (header only)
 - [circular_buffer](https://www.boost.org/doc/libs/1_66_0/doc/html/circular_buffer.html) (header only)
 - [math](https://www.boost.org/doc/libs/1_66_0/libs/math/doc/html/index.html) (header only)
+- [stacktrace](https://www.boost.org/doc/libs/1_66_0/doc/html/stacktrace.html) (header only)
 
 Events and Runtime:
 

lib/base/application.cpp

@@ -19,6 +19,7 @@
 #include <boost/exception/errinfo_api_function.hpp>
 #include <boost/exception/errinfo_errno.hpp>
 #include <boost/exception/errinfo_file_name.hpp>
+#include <boost/stacktrace.hpp>
 #include <sstream>
 #include <iostream>
 #include <fstream>
@@ -34,6 +35,14 @@
 
 using namespace icinga;
 
+#ifdef _WIN32
+/* MSVC throws unhandled C++ exceptions as SEH exceptions with this specific error code.
+ * There seems to be no system header that actually defines this constant.
+ * See also https://devblogs.microsoft.com/oldnewthing/20160915-00/?p=94316
+ */
+#define EXCEPTION_CODE_CXX_EXCEPTION 0xe06d7363
+#endif /* _WIN32 */
+
 REGISTER_TYPE(Application);
 
 boost::signals2::signal<void ()> Application::OnReopenLogs;
@@ -55,6 +64,10 @@ double Application::m_StartTime;
 bool Application::m_ScriptDebuggerEnabled = false;
 double Application::m_LastReloadFailed;
 
+#ifdef _WIN32
+static LPTOP_LEVEL_EXCEPTION_FILTER l_DefaultUnhandledExceptionFilter = nullptr;
+#endif /* _WIN32 */
+
 /**
  * Constructor for the Application class.
  */
@@ -746,11 +759,12 @@ void Application::SigAbrtHandler(int)
 		Log(LogCritical, "Application")
 			<< "Icinga 2 has terminated unexpectedly. Additional information can be found in '" << fname << "'" << "\n";
 
+		ofs << "Caught SIGABRT.\n"
+			<< "Current time: " << Utility::FormatDateTime("%Y-%m-%d %H:%M:%S %z", Utility::GetTime()) << "\n\n";
+
 		DisplayInfoMessage(ofs);
 
-		StackTrace trace;
-		ofs << "Stacktrace:" << "\n";
-		trace.Print(ofs, 1);
+		ofs << "\nStacktrace:\n" << StackTraceFormatter(boost::stacktrace::stacktrace()) << "\n";
 
 		DisplayBugMessage(ofs);
 
@@ -853,9 +867,8 @@ void Application::ExceptionHandler()
 		std::ofstream ofs;
 		ofs.open(fname.CStr());
 
-		ofs << "Caught unhandled exception." << "\n"
-			<< "Current time: " << Utility::FormatDateTime("%Y-%m-%d %H:%M:%S %z", Utility::GetTime()) << "\n"
-			<< "\n";
+		ofs << "Caught unhandled exception.\n"
+			<< "Current time: " << Utility::FormatDateTime("%Y-%m-%d %H:%M:%S %z", Utility::GetTime()) << "\n\n";
 
 		DisplayInfoMessage(ofs);
 
@@ -867,8 +880,19 @@ void Application::ExceptionHandler()
 				<< "\n"
 				<< "Additional information is available in '" << fname << "'" << "\n";
 
+			/* On platforms where HAVE_CXXABI_H is defined, we prefer to print the stack trace that was saved
+			 * when the last exception was thrown. Everywhere else, we do not have this information so we
+			 * collect a stack trace here, which might lack some information, for example when an exception
+			 * is rethrown, but this is still better than nothing.
+			 */
+			boost::stacktrace::stacktrace *stack = nullptr;
+#ifndef HAVE_CXXABI_H
+			boost::stacktrace::stacktrace local_stack;
+			stack = &local_stack;
+#endif /* HAVE_CXXABI_H */
+
 			ofs << "\n"
-				<< DiagnosticInformation(ex)
+				<< DiagnosticInformation(ex, true, stack)
 				<< "\n";
 		}
 
@@ -885,6 +909,15 @@ void Application::ExceptionHandler()
 #ifdef _WIN32
 LONG CALLBACK Application::SEHUnhandledExceptionFilter(PEXCEPTION_POINTERS exi)
 {
+	/* If an unhandled C++ exception occurs with both a termination handler (std::set_terminate()) and an unhandled
+	 * SEH filter (SetUnhandledExceptionFilter()) set, the latter one is called. However, our termination handler is
+	 * better suited for dealing with C++ exceptions. In this case, the SEH exception will have a specific code and
+	 * we can just call the default filter function which will take care of calling the termination handler.
+	 */
+	if (exi->ExceptionRecord->ExceptionCode == EXCEPTION_CODE_CXX_EXCEPTION) {
+		return l_DefaultUnhandledExceptionFilter(exi);
+	}
+
 	if (l_InExceptionHandler)
 		return EXCEPTION_CONTINUE_SEARCH;
 
@@ -909,15 +942,20 @@ LONG CALLBACK Application::SEHUnhandledExceptionFilter(PEXCEPTION_POINTERS exi)
 	Log(LogCritical, "Application")
 		<< "Icinga 2 has terminated unexpectedly. Additional information can be found in '" << fname << "'";
 
+	ofs << "Caught unhandled SEH exception.\n"
+		<< "Current time: " << Utility::FormatDateTime("%Y-%m-%d %H:%M:%S %z", Utility::GetTime()) << "\n\n";
+
 	DisplayInfoMessage(ofs);
 
-	ofs << "Caught unhandled SEH exception." << "\n"
-		<< "Current time: " << Utility::FormatDateTime("%Y-%m-%d %H:%M:%S %z", Utility::GetTime()) << "\n"
-		<< "\n";
+	std::ios::fmtflags savedflags(ofs.flags());
+	ofs << std::showbase << std::hex
+		<< "\nSEH exception:\n"
+		<< "  Code: "    << exi->ExceptionRecord->ExceptionCode    << "\n"
+		<< "  Address: " << exi->ExceptionRecord->ExceptionAddress << "\n"
+		<< "  Flags: "   << exi->ExceptionRecord->ExceptionFlags   << "\n";
+	ofs.flags(savedflags);
 
-	StackTrace trace(exi);
-	ofs << "Stacktrace:" << "\n";
-	trace.Print(ofs, 1);
+	ofs << "\nStacktrace:\n" << StackTraceFormatter(boost::stacktrace::stacktrace()) << "\n";
 
 	DisplayBugMessage(ofs);
 
@@ -938,7 +976,7 @@ void Application::InstallExceptionHandlers()
 	sa.sa_handler = &Application::SigAbrtHandler;
 	sigaction(SIGABRT, &sa, nullptr);
 #else /* _WIN32 */
-	SetUnhandledExceptionFilter(&Application::SEHUnhandledExceptionFilter);
+	l_DefaultUnhandledExceptionFilter = SetUnhandledExceptionFilter(&Application::SEHUnhandledExceptionFilter);
 #endif /* _WIN32 */
 }
 

lib/base/exception.cpp

@@ -1,6 +1,7 @@
 /* Icinga 2 | (c) 2012 Icinga GmbH | GPLv2+ */
 
 #include "base/exception.hpp"
+#include "base/stacktrace.hpp"
 #include <boost/thread/tss.hpp>
 
 #ifdef _WIN32
@@ -13,7 +14,7 @@
 
 using namespace icinga;
 
-static boost::thread_specific_ptr<StackTrace> l_LastExceptionStack;
+static boost::thread_specific_ptr<boost::stacktrace::stacktrace> l_LastExceptionStack;
 static boost::thread_specific_ptr<ContextTrace> l_LastExceptionContext;
 
 #ifdef HAVE_CXXABI_H
@@ -32,6 +33,14 @@ class libcxx_type_info : public std::type_info
 
 
 #if defined(__GLIBCXX__) || defined(_LIBCPPABI_VERSION)
+/**
+ * Attempts to cast an exception to a destination type
+ *
+ * @param obj Exception to be casted
+ * @param src Type information of obj
+ * @param dst Information of which type to cast to
+ * @return Pointer to the exception if the cast is possible, nullptr otherwise
+ */
 inline void *cast_exception(void *obj, const std::type_info *src, const std::type_info *dst)
 {
 #ifdef __GLIBCXX__
@@ -92,6 +101,13 @@ void icinga::RethrowUncaughtException()
 extern "C"
 void __cxa_throw(void *obj, TYPEINFO_TYPE *pvtinfo, void (*dest)(void *))
 {
+	/* This function overrides an internal function of libstdc++ that is called when a C++ exception is thrown in order
+	 * to capture as much information as possible at that time and then call the original implementation. This
+	 * information includes:
+	 *  - stack trace (for later use in DiagnosticInformation)
+	 *  - context trace (for later use in DiagnosticInformation)
+	 */
+
 	auto *tinfo = static_cast<std::type_info *>(pvtinfo);
 
 	typedef void (*cxa_throw_fn)(void *, std::type_info *, void (*)(void *)) __attribute__((noreturn));
@@ -103,6 +119,7 @@ void __cxa_throw(void *obj, TYPEINFO_TYPE *pvtinfo, void (*dest)(void *))
 	l_LastExceptionDest.reset(new DestCallback(dest));
 #endif /* !defined(__GLIBCXX__) && !defined(_WIN32) */
 
+	// resolve symbol to original implementation of __cxa_throw for the call at the end of this function
 	if (real_cxa_throw == nullptr)
 		real_cxa_throw = (cxa_throw_fn)dlsym(RTLD_NEXT, "__cxa_throw");
 
@@ -112,10 +129,12 @@ void __cxa_throw(void *obj, TYPEINFO_TYPE *pvtinfo, void (*dest)(void *))
 
 	if (!uex) {
 #endif /* NO_CAST_EXCEPTION */
-		StackTrace stack;
+		// save the current stack trace in a thread-local variable
+		boost::stacktrace::stacktrace stack;
 		SetLastExceptionStack(stack);
 
 #ifndef NO_CAST_EXCEPTION
+		// save the current stack trace in the boost exception error info if the exception is a boost::exception
 		if (ex && !boost::get_error_info<StackTraceErrorInfo>(*ex))
 			*ex << StackTraceErrorInfo(stack);
 	}
@@ -125,6 +144,7 @@ void __cxa_throw(void *obj, TYPEINFO_TYPE *pvtinfo, void (*dest)(void *))
 	SetLastExceptionContext(context);
 
 #ifndef NO_CAST_EXCEPTION
+	// save the current context trace in the boost exception error info if the exception is a boost::exception
 	if (ex && !boost::get_error_info<ContextTraceErrorInfo>(*ex))
 		*ex << ContextTraceErrorInfo(context);
 #endif /* NO_CAST_EXCEPTION */
@@ -133,14 +153,14 @@ void __cxa_throw(void *obj, TYPEINFO_TYPE *pvtinfo, void (*dest)(void *))
 }
 #endif /* HAVE_CXXABI_H */
 
-StackTrace *icinga::GetLastExceptionStack()
+boost::stacktrace::stacktrace *icinga::GetLastExceptionStack()
 {
 	return l_LastExceptionStack.get();
 }
 
-void icinga::SetLastExceptionStack(const StackTrace& trace)
+void icinga::SetLastExceptionStack(const boost::stacktrace::stacktrace& trace)
 {
-	l_LastExceptionStack.reset(new StackTrace(trace));
+	l_LastExceptionStack.reset(new boost::stacktrace::stacktrace(trace));
 }
 
 ContextTrace *icinga::GetLastExceptionContext()
@@ -153,7 +173,7 @@ void icinga::SetLastExceptionContext(const ContextTrace& context)
 	l_LastExceptionContext.reset(new ContextTrace(context));
 }
 
-String icinga::DiagnosticInformation(const std::exception& ex, bool verbose, StackTrace *stack, ContextTrace *context)
+String icinga::DiagnosticInformation(const std::exception& ex, bool verbose, boost::stacktrace::stacktrace *stack, ContextTrace *context)
 {
 	std::ostringstream result;
 
@@ -215,43 +235,46 @@ String icinga::DiagnosticInformation(const std::exception& ex, bool verbose, Sta
 	const auto *pex = dynamic_cast<const posix_error *>(&ex);
 
 	if (!uex && !pex && verbose) {
-		const StackTrace *st = boost::get_error_info<StackTraceErrorInfo>(ex);
-
-		if (st) {
-			result << *st;
-		} else {
-			result << std::endl;
-
-			if (!stack)
-				stack = GetLastExceptionStack();
-
-			if (stack)
-				result << *stack;
+		// Print the first of the following stack traces (if any exists)
+		//   1. stack trace from boost exception error information
+		const boost::stacktrace::stacktrace *st = boost::get_error_info<StackTraceErrorInfo>(ex);
+		//   2. stack trace explicitly passed as a parameter
+		if (!st) {
+			st = stack;
+		}
+		//   3. stack trace saved when the last exception was thrown
+		if (!st) {
+			st = GetLastExceptionStack();
+		}
 
+		if (st && !st->empty()) {
+			result << "\nStacktrace:\n" << StackTraceFormatter(*st);
 		}
 	}
 
+	// Print the first of the following context traces (if any exists)
+	//   1. context trace from boost exception error information
 	const ContextTrace *ct = boost::get_error_info<ContextTraceErrorInfo>(ex);
+	//   2. context trace explicitly passed as a parameter
+	if (!ct) {
+		ct = context;
+	}
+	//   3. context trace saved when the last exception was thrown
+	if (!ct) {
+		ct = GetLastExceptionContext();
+	}
 
-	if (ct) {
-		result << *ct;
-	} else {
-		result << std::endl;
-
-		if (!context)
-			context = GetLastExceptionContext();
-
-		if (context)
-			result << *context;
+	if (ct && ct->GetLength() > 0) {
+		result << "\nContext:\n" << *ct;
 	}
 
 	return result.str();
 }
 
 String icinga::DiagnosticInformation(const boost::exception_ptr& eptr, bool verbose)
 {
-	StackTrace *pt = GetLastExceptionStack();
-	StackTrace stack;
+	boost::stacktrace::stacktrace *pt = GetLastExceptionStack();
+	boost::stacktrace::stacktrace stack;
 
 	ContextTrace *pc = GetLastExceptionContext();
 	ContextTrace context;