CoreCLR initialization failures logging (#78790)

* CoreCLR initialization failures logging

This change adds detecting and logging of failures during coreclr
initialization. For logging, it uses a new host API
`coreclr_set_error_writer` to register a callback to report the errors
to the host. The hosts have support for optional usage of this API so
that they can work with older runtime versions as well.

The logging checks and reports failures with:
* System.Private.CoreLib.dll
* GC initialization
* JIT initialization
* libSystem.Native.so/dylib on Unix

The logging messages should allow customers to self-diagnose the issues
causing the failures.

* Reflect PR feedback and a bit of cleanup

* revert the try_get_export to not to have the isOptional argument
* Unify the error logging in JIT loading to single variadic macro
* Also fixes Unix build, the code in gcenv.unix.cpp cannot use the GCToEEInterface

* Add single file host detection

* Fix libSystem.Native check location

* Add the coreclr_set_error_writer API to Mono

* Fix Mono x86 missing calling convention spec

* Fix error code returned when the S.P.C. is not found

Author: janvorli

src/coreclr/binder/utils.cpp

@@ -150,56 +150,68 @@ namespace BINDER_SPACE
         isNativeImage = false;
 
         HRESULT pathResult = S_OK;
-        IF_FAIL_GO(pathResult = GetNextPath(paths, startPos, outPath));
-        if (pathResult == S_FALSE)
+        while(true)
         {
-            return S_FALSE;
-        }
-
-        if (Path::IsRelative(outPath))
-        {
-            GO_WITH_HRESULT(E_INVALIDARG);
-        }
-
-        {
-            // Find the beginning of the simple name
-            SString::CIterator iSimpleNameStart = outPath.End();
-
-            if (!outPath.FindBack(iSimpleNameStart, DIRECTORY_SEPARATOR_CHAR_W))
-            {
-                iSimpleNameStart = outPath.Begin();
-            }
-            else
+            IF_FAIL_GO(pathResult = GetNextPath(paths, startPos, outPath));
+            if (pathResult == S_FALSE)
             {
-                // Advance past the directory separator to the first character of the file name
-                iSimpleNameStart++;
+                return S_FALSE;
             }
 
-            if (iSimpleNameStart == outPath.End())
+            if (Path::IsRelative(outPath))
             {
                 GO_WITH_HRESULT(E_INVALIDARG);
             }
 
-            const SString sNiDll(SString::Literal, W(".ni.dll"));
-            const SString sNiExe(SString::Literal, W(".ni.exe"));
-            const SString sDll(SString::Literal, W(".dll"));
-            const SString sExe(SString::Literal, W(".exe"));
-
-            if (!dllOnly && (outPath.EndsWithCaseInsensitive(sNiDll) ||
-                outPath.EndsWithCaseInsensitive(sNiExe)))
-            {
-                simpleName.Set(outPath, iSimpleNameStart, outPath.End() - 7);
-                isNativeImage = true;
-            }
-            else if (outPath.EndsWithCaseInsensitive(sDll) ||
-                (!dllOnly && outPath.EndsWithCaseInsensitive(sExe)))
-            {
-                simpleName.Set(outPath, iSimpleNameStart, outPath.End() - 4);
-            }
-            else
             {
-                // Invalid filename
-                GO_WITH_HRESULT(E_INVALIDARG);
+                // Find the beginning of the simple name
+                SString::CIterator iSimpleNameStart = outPath.End();
+
+                if (!outPath.FindBack(iSimpleNameStart, DIRECTORY_SEPARATOR_CHAR_W))
+                {
+                    iSimpleNameStart = outPath.Begin();
+                }
+                else
+                {
+                    // Advance past the directory separator to the first character of the file name
+                    iSimpleNameStart++;
+                }
+
+                if (iSimpleNameStart == outPath.End())
+                {
+                    GO_WITH_HRESULT(E_INVALIDARG);
+                }
+
+                const SString sNiDll(SString::Literal, W(".ni.dll"));
+                const SString sNiExe(SString::Literal, W(".ni.exe"));
+                const SString sDll(SString::Literal, W(".dll"));
+                const SString sExe(SString::Literal, W(".exe"));
+
+                if (dllOnly && (outPath.EndsWithCaseInsensitive(sExe) ||
+                    outPath.EndsWithCaseInsensitive(sNiExe)))
+                {
+                    // Skip exe files when the caller requested only dlls
+                    continue;
+                }
+
+                if (outPath.EndsWithCaseInsensitive(sNiDll) ||
+                    outPath.EndsWithCaseInsensitive(sNiExe))
+                {
+                    simpleName.Set(outPath, iSimpleNameStart, outPath.End() - 7);
+                    isNativeImage = true;
+                }
+                else if (outPath.EndsWithCaseInsensitive(sDll) ||
+                    outPath.EndsWithCaseInsensitive(sExe))
+                {
+                    simpleName.Set(outPath, iSimpleNameStart, outPath.End() - 4);
+                }
+                else
+                {
+                    // Invalid filename
+                    GO_WITH_HRESULT(E_INVALIDARG);
+                }
+
+                break;
             }
         }
 

src/coreclr/dlls/mscoree/exports.cpp

@@ -181,6 +181,26 @@ static void ConvertConfigPropertiesToUnicode(
     *propertyValuesWRef = propertyValuesW;
 }
 
+coreclr_error_writer_callback_fn g_errorWriter = nullptr;
+
+//
+// Set callback for writing error logging
+//
+// Parameters:
+//  errorWriter             - callback that will be called for each line of the error info
+//                          - passing in NULL removes a callback that was previously set
+//
+// Returns:
+//  S_OK
+//
+extern "C"
+DLLEXPORT
+int coreclr_set_error_writer(coreclr_error_writer_callback_fn error_writer)
+{
+    g_errorWriter = error_writer;
+    return S_OK;
+}
+
 #ifdef FEATURE_GDBJIT
 GetInfoForMethodDelegate getInfoForMethodDelegate = NULL;
 extern "C" int coreclr_create_delegate(void*, unsigned int, const char*, const char*, const char*, void**);
@@ -462,3 +482,16 @@ int coreclr_execute_assembly(
 
     return hr;
 }
+
+void LogErrorToHost(const char* format, ...)
+{
+    if (g_errorWriter != NULL)
+    {
+        char messageBuffer[1024];
+        va_list args;
+        va_start(args, format);
+        _vsnprintf_s(messageBuffer, ARRAY_SIZE(messageBuffer), _TRUNCATE, format, args);
+        g_errorWriter(messageBuffer);
+        va_end(args);
+    }
+}

src/coreclr/dlls/mscoree/mscorwks_ntdef.src

@@ -22,6 +22,7 @@ EXPORTS
         coreclr_create_delegate
         coreclr_execute_assembly
         coreclr_initialize
+        coreclr_set_error_writer
         coreclr_shutdown
         coreclr_shutdown_2
 

src/coreclr/dlls/mscoree/mscorwks_unixexports.src

@@ -2,6 +2,7 @@
 coreclr_create_delegate
 coreclr_execute_assembly
 coreclr_initialize
+coreclr_set_error_writer
 coreclr_shutdown
 coreclr_shutdown_2
 

src/coreclr/gc/env/gcenv.ee.h

@@ -94,6 +94,8 @@ class GCToEEInterface
     static uint32_t GetCurrentProcessCpuCount();
 
     static void DiagAddNewRegion(int generation, uint8_t* rangeStart, uint8_t* rangeEnd, uint8_t* rangeEndReserved);
+
+    static void LogErrorToHost(const char *message);
 };
 
 #endif // __GCENV_EE_H__

src/coreclr/gc/gc.cpp

@@ -13575,13 +13575,17 @@ HRESULT gc_heap::initialize_gc (size_t soh_segment_size,
         gc_log = CreateLogFile(GCConfig::GetLogFile(), false);
 
         if (gc_log == NULL)
+        {
+            GCToEEInterface::LogErrorToHost("Cannot create log file");
             return E_FAIL;
+        }
 
         // GCLogFileSize in MBs.
         gc_log_file_size = static_cast<size_t>(GCConfig::GetLogFileSize());
 
         if (gc_log_file_size <= 0 || gc_log_file_size > 500)
         {
+            GCToEEInterface::LogErrorToHost("Invalid log file size (valid size needs to be larger than 0 and smaller than 500)");
             fclose (gc_log);
             return E_FAIL;
         }
@@ -13591,7 +13595,7 @@ HRESULT gc_heap::initialize_gc (size_t soh_segment_size,
         if (!gc_log_buffer)
         {
             fclose(gc_log);
-            return E_FAIL;
+            return E_OUTOFMEMORY;
         }
 
         memset (gc_log_buffer, '*', gc_log_buffer_size);
@@ -13606,13 +13610,16 @@ HRESULT gc_heap::initialize_gc (size_t soh_segment_size,
         gc_config_log = CreateLogFile(GCConfig::GetConfigLogFile(), true);
 
         if (gc_config_log == NULL)
+        {
+            GCToEEInterface::LogErrorToHost("Cannot create log file");
             return E_FAIL;
+        }
 
         gc_config_log_buffer = new (nothrow) uint8_t [gc_config_log_buffer_size];
         if (!gc_config_log_buffer)
         {
             fclose(gc_config_log);
-            return E_FAIL;
+            return E_OUTOFMEMORY;
         }
 
         compact_ratio = static_cast<int>(GCConfig::GetCompactRatio());
@@ -13739,6 +13746,7 @@ HRESULT gc_heap::initialize_gc (size_t soh_segment_size,
     else
     {
         assert (!"cannot use regions without specifying the range!!!");
+        GCToEEInterface::LogErrorToHost("Cannot use regions without specifying the range (using DOTNET_GCRegionRange)");
         return E_FAIL;
     }
 #else //USE_REGIONS
@@ -13862,6 +13870,7 @@ HRESULT gc_heap::initialize_gc (size_t soh_segment_size,
 
     if (!init_semi_shared())
     {
+        GCToEEInterface::LogErrorToHost("PER_HEAP_ISOLATED data members initialization failed");
         hres = E_FAIL;
     }
 
@@ -45556,6 +45565,7 @@ HRESULT GCHeap::Initialize()
 
     if (!WaitForGCEvent->CreateManualEventNoThrow(TRUE))
     {
+        GCToEEInterface::LogErrorToHost("Creation of WaitForGCEvent failed");
         return E_FAIL;
     }
 
@@ -45637,9 +45647,15 @@ HRESULT GCHeap::Initialize()
         int hb_info_size_per_node = hb_info_size_per_proc * procs_per_numa_node;
         uint8_t* numa_mem = (uint8_t*)GCToOSInterface::VirtualReserve (hb_info_size_per_node, 0, 0, numa_node_index);
         if (!numa_mem)
+        {
+            GCToEEInterface::LogErrorToHost("Reservation of numa_mem failed");
             return E_FAIL;
+        }
         if (!GCToOSInterface::VirtualCommit (numa_mem, hb_info_size_per_node, numa_node_index))
+        {
+            GCToEEInterface::LogErrorToHost("Commit of numa_mem failed");
             return E_FAIL;
+        }
 
         heap_balance_info_proc* hb_info_procs = (heap_balance_info_proc*)numa_mem;
         hb_info_numa_nodes[numa_node_index].hb_info_procs = hb_info_procs;

src/coreclr/gc/gccommon.cpp

@@ -17,7 +17,7 @@ IGCHeapInternal* g_theGCHeap;
 IGCHandleManager* g_theGCHandleManager;
 
 #ifdef BUILD_AS_STANDALONE
-IGCToCLR* g_theGCToCLR;
+IGCToCLR2* g_theGCToCLR;
 VersionInfo g_runtimeSupportedVersion;
 #endif // BUILD_AS_STANDALONE
 

src/coreclr/gc/gcenv.ee.standalone.inl

@@ -9,7 +9,7 @@
 
 // The singular interface instance. All calls in GCToEEInterface
 // will be forwarded to this interface instance.
-extern IGCToCLR* g_theGCToCLR;
+extern IGCToCLR2* g_theGCToCLR;
 
 // GC version that the current runtime supports
 extern VersionInfo g_runtimeSupportedVersion;
@@ -314,4 +314,12 @@ inline void GCToEEInterface::DiagAddNewRegion(int generation, uint8_t* rangeStar
     g_theGCToCLR->DiagAddNewRegion(generation, rangeStart, rangeEnd, rangeEndReserved);
 }
 
+inline void GCToEEInterface::LogErrorToHost(const char *message)
+{
+    if (g_runtimeSupportedVersion.MajorVersion >= GC_INTERFACE2_MAJOR_VERSION)
+    {
+        g_theGCToCLR->LogErrorToHost(message);
+    }
+}
+
 #endif // __GCTOENV_EE_STANDALONE_INL__

src/coreclr/gc/gcinterface.ee.h

@@ -448,4 +448,11 @@ class IGCToCLR {
     void DiagAddNewRegion(int generation, uint8_t* rangeStart, uint8_t* rangeEnd, uint8_t* rangeEndReserved) = 0;
 };
 
+class IGCToCLR2 : public IGCToCLR {
+public:
+
+    virtual
+    void LogErrorToHost(const char *message) = 0;
+};
+
 #endif // _GCINTERFACE_EE_H_

src/coreclr/gc/gcinterface.h

@@ -6,13 +6,17 @@
 
 // The major version of the GC/EE interface. Breaking changes to this interface
 // require bumps in the major version number.
-#define GC_INTERFACE_MAJOR_VERSION 5
+#define GC_INTERFACE_MAJOR_VERSION 6
 
 // The minor version of the GC/EE interface. Non-breaking changes are required
 // to bump the minor version number. GCs and EEs with minor version number
 // mismatches can still interopate correctly, with some care.
 #define GC_INTERFACE_MINOR_VERSION 1
 
+// The major version of the GC/EE interface. Breaking changes to this interface
+// require bumps in the major version number.
+#define GC_INTERFACE2_MAJOR_VERSION 6
+
 struct ScanContext;
 struct gc_alloc_context;
 class CrawlFrame;