Add XUnitLogChecker to log libraries dumps (#93906)

Libraries CI failures that generate dumps will now show the dump output in the console log itself.

Author: carlossanlop

Directory.Build.props

@@ -180,7 +180,6 @@
     <TestExclusionListTasksAssemblyPath>$([MSBuild]::NormalizePath('$(TestExclusionListTasksDir)', 'TestExclusionListTasks.dll'))</TestExclusionListTasksAssemblyPath>
     <CoreCLRToolPath>$([MSBuild]::NormalizeDirectory('$(ArtifactsBinDir)', 'coreclr', '$(TargetOS).$(TargetArchitecture).$(RuntimeConfiguration)'))</CoreCLRToolPath>
     <ILAsmToolPath Condition="'$(DotNetBuildFromSource)' == 'true' or '$(BuildArchitecture)' == 's390x' or '$(BuildArchitecture)' == 'ppc64le'">$(CoreCLRToolPath)</ILAsmToolPath>
-
     <WasmtimeDir Condition="'$(WasmtimeDir)' == '' and '$(WASMTIME_PATH)' != '' and Exists($(WASMTIME_PATH))">$(WASMTIME_PATH)</WasmtimeDir>
     <WasmtimeDir Condition="'$(WasmtimeDir)' == ''">$([MSBuild]::NormalizeDirectory($(ArtifactsObjDir), 'wasmtime'))</WasmtimeDir>
     <InstallWasmtimeForTests Condition="'$(InstallWasmtimeForTests)' == '' and !Exists($(WasmtimeDir))">true</InstallWasmtimeForTests>
@@ -332,6 +331,11 @@
 
     <!-- this property is used by the SDK to pull in mono-based runtime packs -->
     <UseMonoRuntime Condition="'$(UseMonoRuntime)' == '' and '$(RuntimeFlavor)' == 'Mono'">true</UseMonoRuntime>
+    
+    <!-- For enabling the use of XUnitLogChecker in coreclr and libraries test runs. -->
+    <IsXUnitLogCheckerSupported Condition="'$(IsXUnitLogCheckerSupported)' == ''">false</IsXUnitLogCheckerSupported>
+    <IsXUnitLogCheckerSupported Condition="'$(RuntimeFlavor)' == 'CoreCLR' and '$(TestNativeAot)' != 'true' and '$(TestRunNamePrefixSuffix)' != 'NativeAOT_Release' and '$(TargetOS)' != 'browser' and '$(TargetOS)' != 'wasi' and '$(TargetOS)' != 'ios' and '$(TargetOS)' != 'iossimulator' and '$(TargetOS)' != 'tvos' and '$(TargetOS)' != 'tvossimulator' and '$(TargetOS)' != 'maccatalyst' and '$(TargetOS)' != 'android'">true</IsXUnitLogCheckerSupported>
+    <XUnitLogCheckerLibrariesOutDir>$([MSBuild]::NormalizeDirectory('$(ArtifactsBinDir)', 'XUnitLogChecker'))</XUnitLogCheckerLibrariesOutDir>
   </PropertyGroup>
 
   <!-- Packaging -->

eng/pipelines/libraries/prepare-for-bin-publish.yml

@@ -14,6 +14,7 @@ steps:
           ref/**
           runtime/**
           testhost/**
+          XUnitLogChecker/**
 
   - task: CopyFiles@2
     displayName: Prepare bin folders to publish

eng/testing/RunnerTemplate.cmd

@@ -92,6 +92,41 @@ if %_exit_code%==1 (
     )
   )
 )
+
+if NOT "%__IsXUnitLogCheckerSupported%"=="1" (
+  echo XUnitLogChecker not supported for this test case. Skipping.
+  GOTO SKIP_XUNITLOGCHECKER
+)
+
+echo ----- start ===============  XUnitLogChecker Output =====================================================
+
+set DOTNET_EXE=%RUNTIME_PATH%\dotnet.exe
+set XUNITLOGCHECKER_DLL=%HELIX_CORRELATION_PAYLOAD%\XUnitLogChecker.dll
+set XUNITLOGCHECKER_COMMAND=%DOTNET_EXE% --roll-forward Major %XUNITLOGCHECKER_DLL% --dumps-path %HELIX_DUMP_FOLDER%
+set XUNITLOGCHECKER_EXIT_CODE=1
+
+if NOT EXIST %DOTNET_EXE% (
+  echo dotnet.exe does not exist in the expected location: %DOTNET_EXE%
+  GOTO XUNITLOGCHECKER_END
+) else if NOT EXIST %XUNITLOGCHECKER_DLL% (
+  echo XUnitLogChecker.dll does not exist in the expected location: %XUNITLOGCHECKER_DLL%
+  GOTO XUNITLOGCHECKER_END
+)
+
+echo %XUNITLOGCHECKER_COMMAND%
+%XUNITLOGCHECKER_COMMAND%
+set XUNITLOGCHECKER_EXIT_CODE=%ERRORLEVEL%
+
+:XUNITLOGCHECKER_END
+
+if %XUNITLOGCHECKER_EXIT_CODE% NEQ 0 (
+  set _exit_code=%XUNITLOGCHECKER_EXIT_CODE%
+)
+
+echo ----- end ===============  XUnitLogChecker Output - exit code %XUNITLOGCHECKER_EXIT_CODE% ===============
+
+:SKIP_XUNITLOGCHECKER
+
 exit /b %_exit_code%
 :: ========================= END Test Execution =================================
 

eng/testing/RunnerTemplate.sh

@@ -54,80 +54,60 @@ exitcode_list[131]="SIGQUIT Ctrl-\ occurred. Core dumped."
 exitcode_list[132]="SIGILL  Illegal Instruction. Core dumped. Likely codegen issue."
 exitcode_list[133]="SIGTRAP Breakpoint hit. Core dumped."
 exitcode_list[134]="SIGABRT Abort. Managed or native assert, or runtime check such as heap corruption, caused call to abort(). Core dumped."
-exitcode_list[135]="IGBUS  Unaligned memory access. Core dumped."
+exitcode_list[135]="IGBUS   Unaligned memory access. Core dumped."
 exitcode_list[136]="SIGFPE  Bad floating point arguments. Core dumped."
 exitcode_list[137]="SIGKILL Killed eg by kill"
 exitcode_list[139]="SIGSEGV Illegal memory access. Deref invalid pointer, overrunning buffer, stack overflow etc. Core dumped."
 exitcode_list[143]="SIGTERM Terminated. Usually before SIGKILL."
 exitcode_list[159]="SIGSYS  Bad System Call."
 
-function print_info_from_core_file_using_lldb {
+function move_core_file_to_temp_location {
   local core_file_name=$1
-  local executable_name=$2
-  local plugin_path_name="$RUNTIME_PATH/shared/Microsoft.NETCore.App/9.9.9/libsosplugin.so"
 
-  # check for existence of lldb on the path
-  hash lldb 2>/dev/null || { echo >&2 "lldb was not found. Unable to print core file."; return; }
+  # Append the dmp extension to ensure XUnitLogChecker finds it
+  local new_location=$HELIX_DUMP_FOLDER/$core_file_name.dmp
 
-  # pe, clrstack, and dumpasync are defined in libsosplugin.so
-  if [ ! -f $plugin_path_name ]; then
-    echo $plugin_path_name cannot be found.
-    return
-  fi
-
-  echo ----- start ===============  lldb Output =====================================================
-  echo Printing managed exceptions, managed call stacks, and async state machines.
-  lldb -O "settings set target.exec-search-paths $RUNTIME_PATH" -o "plugin load $plugin_path_name" -o "clrthreads -managedexception" -o "pe -nested" -o "clrstack -all -a -f" -o "dumpasync -fields -stacks -roots" -o "quit"  --core $core_file_name $executable_name
-  echo ----- end ===============  lldb Output =======================================================
-}
-
-function print_info_from_core_file_using_gdb {
-  local core_file_name=$1
-  local executable_name=$2
-
-  # Check for the existence of GDB on the path
-  hash gdb 2>/dev/null || { echo >&2 "GDB was not found. Unable to print core file."; return; }
+  echo "Copying dump file '$core_file_name' to '$new_location'"
+  cp $core_file_name $new_location
 
-  echo ----- start ===============  GDB Output =====================================================
-  # Open the dump in GDB and print the stack from each thread. We can add more
-  # commands here if desired.
-  echo printing native stack.
-  gdb --batch -ex "thread apply all bt full" -ex "quit" $executable_name $core_file_name
-  echo ----- end ===============  GDB Output =======================================================
+  # Delete the old one
+  rm $core_file_name
 }
 
-function print_info_from_core_file {
-  local core_file_name=$1
-  local executable_name=$RUNTIME_PATH/$2
-
-  if ! [ -e $executable_name ]; then
-    echo "Unable to find executable $executable_name"
-    return
-  elif ! [ -e $core_file_name ]; then
-    echo "Unable to find core file $core_file_name"
-    return
+xunitlogchecker_exit_code=0
+function invoke_xunitlogchecker {
+  local dump_folder=$1
+
+  total_dumps=$(find $dump_folder -name "*.dmp" | wc -l)
+  
+  if [[ $total_dumps > 0 ]]; then
+    echo "Total dumps found in $dump_folder: $total_dumps"
+    xunitlogchecker_file_name="$HELIX_CORRELATION_PAYLOAD/XUnitLogChecker.dll"
+    dotnet_file_name="$RUNTIME_PATH/dotnet"
+
+    if [[ ! -f $dotnet_file_name ]]; then
+      echo "'$dotnet_file_name' was not found. Unable to run XUnitLogChecker."
+      xunitlogchecker_exit_code=1
+    elif [[ ! -f $xunitlogchecker_file_name ]]; then 
+      echo "'$xunitlogchecker_file_name' was not found. Unable to print dump file contents."
+      xunitlogchecker_exit_code=2
+    elif [[ ! -d $dump_folder ]]; then
+      echo "The dump directory '$dump_folder' does not exist."
+    else
+      echo "Executing XUnitLogChecker in $dump_folder..."
+      cmd="$dotnet_file_name --roll-forward Major $xunitlogchecker_file_name --dumps-path $dump_folder"
+      echo "$cmd"
+      $cmd
+      xunitlogchecker_exit_code=$?
+    fi
+  else
+    echo "No dumps found in $dump_folder."
   fi
-  echo "Printing info from core file $core_file_name"
-  print_info_from_core_file_using_gdb $core_file_name $executable_name
-  print_info_from_core_file_using_lldb $core_file_name $executable_name
-}
-
-function copy_core_file_to_temp_location {
-  local core_file_name=$1
-
-  local storage_location="/tmp/coredumps"
-
-  # Create the directory (this shouldn't fail even if it already exists).
-  mkdir -p $storage_location
-
-  local new_location=$storage_location/core.$RANDOM
-
-  echo "Copying core file $core_file_name to $new_location in case you need it."
-  cp $core_file_name $new_location
 }
 
 # ========================= BEGIN Core File Setup ============================
-if [[ "$(uname -s)" == "Darwin" ]]; then
+system_name="$(uname -s)"
+if [[ $system_name == "Darwin" ]]; then
   # On OS X, we will enable core dump generation only if there are no core
   # files already in /cores/ at this point. This is being done to prevent
   # inadvertently flooding the CI machines with dumps.
@@ -140,21 +120,11 @@ if [[ "$(uname -s)" == "Darwin" ]]; then
     #   https://github.com/dotnet/core-eng/issues/15597
     ulimit -c 0
   fi
-
-elif [[ "$(uname -s)" == "Linux" ]]; then
-  # On Linux, we'll enable core file generation unconditionally, and if a dump
-  # is generated, we will print some useful information from it and delete the
-  # dump immediately.
-
-  if [ -e /proc/self/coredump_filter ]; then
-      # Include memory in private and shared file-backed mappings in the dump.
-      # This ensures that we can see disassembly from our shared libraries when
-      # inspecting the contents of the dump. See 'man core' for details.
-      echo -n 0x3F > /proc/self/coredump_filter
-  fi
-
-  ulimit -c unlimited
 fi
+
+export DOTNET_DbgEnableMiniDump=1
+export DOTNET_EnableCrashReport=1
+export DOTNET_DbgMiniDumpName=$HELIX_DUMP_FOLDER/coredump.%d.dmp
 # ========================= END Core File Setup ==============================
 
 # ========================= BEGIN support for SuperPMI collection ==============================
@@ -171,7 +141,7 @@ if [ ! -z $spmi_enable_collection ]; then
   fi
   mkdir -p $spmi_collect_dir
   export spmi_file_extension=so
-  if [[ "$(uname -s)" == "Darwin" ]]; then
+  if [[ $system_name == "Darwin" ]]; then
     export spmi_file_extension=dylib
   fi
   export SuperPMIShimLogPath=$spmi_collect_dir
@@ -220,51 +190,59 @@ if [[ $test_exitcode -ne 0 ]]; then
   echo ulimit -c value: $(ulimit -c)
 fi
 
-if [[ "$(uname -s)" == "Linux" && $test_exitcode -ne 0 ]]; then
-  if [ -n "$HELIX_WORKITEM_PAYLOAD" ]; then
-
-    # For abrupt failures, in Helix, dump some of the kernel log, in case there is a hint
-    if [[ $test_exitcode -ne 1 ]]; then
-      dmesg | tail -50
-    fi
-
-    have_sleep=$(which sleep)
-    if [ -x "$have_sleep" ]; then
-      echo Waiting a few seconds for any dump to be written..
-      sleep 10s
-    fi
-  fi
-
+if [[ $system_name == "Linux" && $test_exitcode -ne 0 ]]; then
   echo cat /proc/sys/kernel/core_pattern: $(cat /proc/sys/kernel/core_pattern)
   echo cat /proc/sys/kernel/core_uses_pid: $(cat /proc/sys/kernel/core_uses_pid)
   echo cat /proc/sys/kernel/coredump_filter: $(cat /proc/sys/kernel/coredump_filter)
 
-  echo Looking around for any Linux dump..
-
   # Depending on distro/configuration, the core files may either be named "core"
   # or "core.<PID>" by default. We read /proc/sys/kernel/core_uses_pid to
   # determine which it is.
   core_name_uses_pid=0
   if [[ -e /proc/sys/kernel/core_uses_pid && "1" == $(cat /proc/sys/kernel/core_uses_pid) ]]; then
     core_name_uses_pid=1
   fi
+  
+  # The osx dumps are too large to egress the machine
+  echo Looking around for any Linux dumps...
 
   if [[ "$core_name_uses_pid" == "1" ]]; then
     # We don't know what the PID of the process was, so let's look at all core
     # files whose name matches core.NUMBER
-    echo Looking for files matching core.* ...
-    for f in core.*; do
-      [[ $f =~ core.[0-9]+ ]] && print_info_from_core_file "$f" "dotnet" && copy_core_file_to_temp_location "$f" && rm "$f"
+    echo "Looking for files matching core.* ..."
+    for f in $(find . -name "core.*"); do
+      [[ $f =~ core.[0-9]+ ]] && move_core_file_to_temp_location "$f"
     done
-  elif [ -f core ]; then
-    echo found a dump named core in $EXECUTION_DIR !
-    print_info_from_core_file "core" "dotnet"
-    copy_core_file_to_temp_location "core"
-    rm "core"
-  else
-    echo ... found no dump in $PWD
   fi
+
+  if [ -f core ]; then
+    move_core_file_to_temp_location "core"
+  fi
+fi
+
+if [ -n "$HELIX_WORKITEM_PAYLOAD" ]; then
+  # For abrupt failures, in Helix, dump some of the kernel log, in case there is a hint
+  if [[ $test_exitcode -ne 1 ]]; then
+    dmesg | tail -50
+  fi
+
 fi
+
+if [[ -z "$__IsXUnitLogCheckerSupported" ]]; then
+  echo "The '__IsXUnitLogCheckerSupported' env var is not set."
+elif [[ "$__IsXUnitLogCheckerSupported" != "1" ]]; then
+  echo "XUnitLogChecker not supported for this test case. Skipping."
+else
+  echo ----- start ===============  XUnitLogChecker Output =====================================================
+  
+  invoke_xunitlogchecker "$HELIX_DUMP_FOLDER"
+
+  if [[ $xunitlogchecker_exit_code -ne 0 ]]; then
+    test_exitcode=$xunitlogchecker_exit_code
+  fi
+  echo ----- end ===============  XUnitLogChecker Output - exit code $xunitlogchecker_exit_code ===========================
+fi
+
 popd >/dev/null
 # ======================== END Core File Inspection ==========================
 # The helix work item should not exit with non-zero if tests ran and produced results