Add faster DAC EnumMemoryRegion option with less memory usage (#74300) (#74464)

Issue: #72148

Instead of drilling down into all the individual MT/MD/EEClass, etc. data structures, add the LoaderAllocator/LoaderHeaps regions directly.

Add new CLRDATA_ENUM_MEM_HEAP2 flag for the fast path.

To reduce risk of incomplete core dumps this is enabled by the COMPlus_EnableFastHeapDumps env var. This env var is only looked at by the Linux/MacOS createdump. It is currently ignored on Windows. The new HEAP2 flag is works when passed to the EnumMemoryRegions API on Windows but createdump can't set it because MiniDumpWriteDump in dbghelp.dll loads/calls the DAC API.

Fix MacOS dlopen error message

Author: Mike McLaughlin

src/coreclr/debug/createdump/crashinfo.cpp

@@ -2,6 +2,7 @@
 // The .NET Foundation licenses this file to you under the MIT license.
 
 #include "createdump.h"
+#include <clrconfignocache.h>
 
 // This is for the PAL_VirtualUnwindOutOfProc read memory adapter.
 CrashInfo* g_crashInfo;
@@ -328,24 +329,34 @@ CrashInfo::EnumerateMemoryRegionsWithDAC(MINIDUMP_TYPE minidumpType)
     {
         TRACE("EnumerateMemoryRegionsWithDAC: Memory enumeration STARTED (%d %d)\n", m_enumMemoryPagesAdded, m_dataTargetPagesAdded);
 
-        // Since on both Linux and MacOS all the RW regions will be added for heap
-        // dumps by createdump, the only thing differentiating a MiniDumpNormal and
-        // a MiniDumpWithPrivateReadWriteMemory is that the later uses the EnumMemory
-        // APIs. This is kind of expensive on larger applications (4 minutes, or even
-        // more), and this should already be in RW pages. Change the dump type to the
-        // faster normal one. This one already ensures necessary DAC globals, etc.
-        // without the costly assembly, module, class, type runtime data structures
-        // enumeration.
+        // Since on MacOS all the RW regions will be added for heap dumps by createdump, the
+        // only thing differentiating a MiniDumpNormal and a MiniDumpWithPrivateReadWriteMemory
+        // is that the later uses the EnumMemoryRegions APIs. This is kind of expensive on larger
+        // applications (4 minutes, or even more), and this should already be in RW pages. Change
+        // the dump type to the faster normal one. This one already ensures necessary DAC globals,
+        // etc. without the costly assembly, module, class, type runtime data structures enumeration.
+        CLRDataEnumMemoryFlags flags = CLRDATA_ENUM_MEM_DEFAULT;
         if (minidumpType & MiniDumpWithPrivateReadWriteMemory)
         {
-            char* fastHeapDumps = getenv("COMPlus_DbgEnableFastHeapDumps");
-            if (fastHeapDumps != nullptr && strcmp(fastHeapDumps, "1") == 0)
+            // This is the old fast heap env var for backwards compatibility for VS4Mac.
+            CLRConfigNoCache fastHeapDumps = CLRConfigNoCache::Get("DbgEnableFastHeapDumps", /*noprefix*/ false, &getenv);
+            DWORD val = 0;
+            if (fastHeapDumps.IsSet() && fastHeapDumps.TryAsInteger(10, val) && val == 1)
             {
                 minidumpType = MiniDumpNormal;
             }
+            // This the new variable that also skips the expensive (in both time and memory usage)
+            // enumeration of the low level data structures and adds all the loader allocator heaps
+            // instead. The above original env var didn't generate a complete enough heap dump on
+            // Linux and this new one does.
+            fastHeapDumps = CLRConfigNoCache::Get("EnableFastHeapDumps", /*noprefix*/ false, &getenv);
+            if (fastHeapDumps.IsSet() && fastHeapDumps.TryAsInteger(10, val) && val == 1)
+            {
+                flags = CLRDATA_ENUM_MEM_HEAP2;
+            }
         }
         // Calls CrashInfo::EnumMemoryRegion for each memory region found by the DAC
-        HRESULT hr = m_pClrDataEnumRegions->EnumMemoryRegions(this, minidumpType, CLRDATA_ENUM_MEM_DEFAULT);
+        HRESULT hr = m_pClrDataEnumRegions->EnumMemoryRegions(this, minidumpType, flags);
         if (FAILED(hr))
         {
             printf_error("EnumMemoryRegions FAILED %s (%08x)\n", GetHResultString(hr), hr);

src/coreclr/debug/createdump/crashinfomac.cpp

@@ -487,7 +487,7 @@ ModuleInfo::LoadModule()
         }
         else
         {
-            TRACE("LoadModule: dlopen(%s) FAILED %d %s\n", m_moduleName.c_str(), errno, strerror(errno));
+            TRACE("LoadModule: dlopen(%s) FAILED %s\n", m_moduleName.c_str(), dlerror());
         }
     }
 }

src/coreclr/debug/daccess/daccess.cpp

@@ -6246,24 +6246,15 @@ bool ClrDataAccess::ReportMem(TADDR addr, TSIZE_T size, bool fExpectSuccess /*=
     {
         if (!IsFullyReadable(addr, size))
         {
-            if (!fExpectSuccess)
+            if (fExpectSuccess)
             {
-                // We know the read might fail (eg. we're trying to find mapped pages in
-                // a module image), so just skip this block silently.
-                // Note that the EnumMemoryRegion callback won't necessarily do anything if any part of
-                // the region is unreadable, and so there is no point in calling it.  For cases where we expect
-                // the read might fail, but we want to report any partial blocks, we have to break up the region
-                // into pages and try reporting each page anyway
-                return true;
+                // We're reporting bogus memory, so the target must be corrupt (or there is a issue). We should abort
+                // reporting and continue with the next data structure (where the exception is caught),
+                // just like we would for a DAC read error (otherwise we might do something stupid
+                // like get into an infinite loop, or otherwise waste time with corrupt data).
+                TARGET_CONSISTENCY_CHECK(false, "Found unreadable memory while reporting memory regions for dump gathering");
+                return false;
             }
-
-            // We're reporting bogus memory, so the target must be corrupt (or there is a issue). We should abort
-            // reporting and continue with the next data structure (where the exception is caught),
-            // just like we would for a DAC read error (otherwise we might do something stupid
-            // like get into an infinite loop, or otherwise waste time with corrupt data).
-
-            TARGET_CONSISTENCY_CHECK(false, "Found unreadable memory while reporting memory regions for dump gathering");
-            return false;
         }
     }
 
@@ -6275,9 +6266,7 @@ bool ClrDataAccess::ReportMem(TADDR addr, TSIZE_T size, bool fExpectSuccess /*=
     // data structure at all.  Hopefully experience will help guide this going forward.
     // @dbgtodo : Extend dump-gathering API to allow a dump-log to be included.
     const TSIZE_T kMaxMiniDumpRegion = 4*1024*1024 - 3;    // 4MB-3
-    if( size > kMaxMiniDumpRegion
-        && (m_enumMemFlags == CLRDATA_ENUM_MEM_MINI
-          || m_enumMemFlags == CLRDATA_ENUM_MEM_TRIAGE))
+    if (size > kMaxMiniDumpRegion && (m_enumMemFlags == CLRDATA_ENUM_MEM_MINI || m_enumMemFlags == CLRDATA_ENUM_MEM_TRIAGE))
     {
         TARGET_CONSISTENCY_CHECK( false, "Dump target consistency failure - truncating minidump data structure");
         size = kMaxMiniDumpRegion;

src/coreclr/debug/daccess/dacdbiimpl.cpp

@@ -3618,7 +3618,7 @@ void DacDbiInterfaceImpl::EnumerateMemRangesForLoaderAllocator(PTR_LoaderAllocat
 
     // GetVirtualCallStubManager returns VirtualCallStubManager*, but it's really an address to target as
     // pLoaderAllocator is DACized. Cast it so we don't try to to a Host to Target translation.
-    VirtualCallStubManager *pVcsMgr = PTR_VirtualCallStubManager(TO_TADDR(pLoaderAllocator->GetVirtualCallStubManager()));
+    VirtualCallStubManager *pVcsMgr = pLoaderAllocator->GetVirtualCallStubManager();
     LOG((LF_CORDB, LL_INFO10000, "DDBII::EMRFLA: VirtualCallStubManager 0x%x\n", PTR_HOST_TO_TADDR(pVcsMgr)));
     if (pVcsMgr)
     {

src/coreclr/debug/daccess/dacimpl.h

@@ -1328,6 +1328,7 @@ class ClrDataAccess
 
     HRESULT EnumMemCollectImages();
     HRESULT EnumMemCLRStatic(CLRDataEnumMemoryFlags flags);
+    HRESULT EnumMemDumpJitManagerInfo(IN CLRDataEnumMemoryFlags flags);
     HRESULT EnumMemCLRHeapCrticalStatic(CLRDataEnumMemoryFlags flags);
     HRESULT EnumMemDumpModuleList(CLRDataEnumMemoryFlags flags);
     HRESULT EnumMemDumpAppDomainInfo(CLRDataEnumMemoryFlags flags);

src/coreclr/debug/daccess/enummem.cpp

@@ -283,6 +283,21 @@ HRESULT ClrDataAccess::EnumMemCLRStatic(IN CLRDataEnumMemoryFlags flags)
     return S_OK;
 }
 
+HRESULT ClrDataAccess::EnumMemDumpJitManagerInfo(IN CLRDataEnumMemoryFlags flags)
+{
+    SUPPORTS_DAC;
+
+    HRESULT status = S_OK;
+
+    if (flags == CLRDATA_ENUM_MEM_HEAP2)
+    {
+        EEJitManager* managerPtr = ExecutionManager::GetEEJitManager();
+        managerPtr->EnumMemoryRegions(flags);
+    }
+
+    return status;
+}
+
 //++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
 //
 // This function reports memory that a heap dump need to debug CLR
@@ -325,6 +340,9 @@ HRESULT ClrDataAccess::EnumMemoryRegionsWorkerHeap(IN CLRDataEnumMemoryFlags fla
     // Dump AppDomain-specific info
     CATCH_ALL_EXCEPT_RETHROW_COR_E_OPERATIONCANCELLED( status = EnumMemDumpAppDomainInfo(flags); )
 
+    // Dump jit manager info
+    CATCH_ALL_EXCEPT_RETHROW_COR_E_OPERATIONCANCELLED( status = EnumMemDumpJitManagerInfo(flags); )
+
     // Dump the Debugger object data needed
     CATCH_ALL_EXCEPT_RETHROW_COR_E_OPERATIONCANCELLED( g_pDebugger->EnumMemoryRegions(flags); )
 
@@ -680,6 +698,11 @@ HRESULT ClrDataAccess::EnumMemDumpAppDomainInfo(CLRDataEnumMemoryFlags flags)
 {
     SUPPORTS_DAC;
 
+    if (flags == CLRDATA_ENUM_MEM_HEAP2)
+    {
+        SystemDomain::System()->GetLoaderAllocator()->EnumMemoryRegions(flags);
+    }
+
     AppDomainIterator adIter(FALSE);
     EX_TRY
     {
@@ -1853,7 +1876,7 @@ HRESULT ClrDataAccess::EnumMemoryRegionsWrapper(IN CLRDataEnumMemoryFlags flags)
             // triage micro-dump
             status = EnumMemoryRegionsWorkerMicroTriage(flags);
         }
-        else if (flags == CLRDATA_ENUM_MEM_HEAP)
+        else if (flags == CLRDATA_ENUM_MEM_HEAP || flags == CLRDATA_ENUM_MEM_HEAP2)
         {
             status = EnumMemoryRegionsWorkerHeap(flags);
         }
@@ -1946,7 +1969,15 @@ ClrDataAccess::EnumMemoryRegions(IN ICLRDataEnumMemoryRegionsCallback* callback,
         if (miniDumpFlags & MiniDumpWithPrivateReadWriteMemory)
         {
             // heap dump
-            status = EnumMemoryRegionsWrapper(CLRDATA_ENUM_MEM_HEAP);
+            if (flags == CLRDATA_ENUM_MEM_HEAP2)
+            {
+                DacLogMessage("EnumMemoryRegions(CLRDATA_ENUM_MEM_HEAP2)\n");
+            }
+            else
+            {
+                flags = CLRDATA_ENUM_MEM_HEAP;
+            }
+            status = EnumMemoryRegionsWrapper(flags);
         }
         else if (miniDumpFlags & MiniDumpWithFullAuxiliaryState)
         {

src/coreclr/debug/daccess/request.cpp

@@ -3497,7 +3497,7 @@ ClrDataAccess::TraverseVirtCallStubHeap(CLRDATA_ADDRESS pAppDomain, VCSHeapType
     SOSDacEnter();
 
     BaseDomain* pBaseDomain = PTR_BaseDomain(TO_TADDR(pAppDomain));
-    VirtualCallStubManager *pVcsMgr = PTR_VirtualCallStubManager((TADDR)pBaseDomain->GetLoaderAllocator()->GetVirtualCallStubManager());
+    VirtualCallStubManager *pVcsMgr = pBaseDomain->GetLoaderAllocator()->GetVirtualCallStubManager();
     if (!pVcsMgr)
     {
         hr = E_POINTER;

src/coreclr/debug/ee/debugger.cpp

@@ -16596,22 +16596,23 @@ Debugger::EnumMemoryRegions(CLRDataEnumMemoryFlags flags)
 {
     DAC_ENUM_VTHIS();
     SUPPORTS_DAC;
-    _ASSERTE(m_rgHijackFunction != NULL);
 
-    if ( flags != CLRDATA_ENUM_MEM_TRIAGE)
+    if (flags != CLRDATA_ENUM_MEM_TRIAGE)
     {
         if (m_pMethodInfos.IsValid())
         {
             m_pMethodInfos->EnumMemoryRegions(flags);
         }
 
-        DacEnumMemoryRegion(dac_cast<TADDR>(m_pLazyData),
-                                sizeof(DebuggerLazyInit));
+        DacEnumMemoryRegion(dac_cast<TADDR>(m_pLazyData), sizeof(DebuggerLazyInit));
     }
 
     // Needed for stack walking from an initial native context.  If the debugger can find the
     // on-disk image of clr.dll, then this is not necessary.
-    DacEnumMemoryRegion(dac_cast<TADDR>(m_rgHijackFunction), sizeof(MemoryRange)*kMaxHijackFunctions);
+    if (m_rgHijackFunction.IsValid())
+    {
+        DacEnumMemoryRegion(dac_cast<TADDR>(m_rgHijackFunction), sizeof(MemoryRange)*kMaxHijackFunctions);
+    }
 }
 
 

src/coreclr/debug/ee/functioninfo.cpp

@@ -2456,9 +2456,7 @@ DebuggerMethodInfoEntry::EnumMemoryRegions(CLRDataEnumMemoryFlags flags)
 
     // For a MiniDumpNormal, what is needed for modules is already enumerated elsewhere.
     // Don't waste time doing it here an extra time. Also, this will add many MB extra into the dump.
-    if ((key.pModule.IsValid()) &&
-        CLRDATA_ENUM_MEM_MINI != flags
-        && CLRDATA_ENUM_MEM_TRIAGE != flags)
+    if ((key.pModule.IsValid()) && CLRDATA_ENUM_MEM_MINI != flags && CLRDATA_ENUM_MEM_TRIAGE != flags && CLRDATA_ENUM_MEM_HEAP2 != flags)
     {
         key.pModule->EnumMemoryRegions(flags, true);
     }
@@ -2476,7 +2474,7 @@ DebuggerMethodInfo::EnumMemoryRegions(CLRDataEnumMemoryFlags flags)
     DAC_ENUM_DTHIS();
     SUPPORTS_DAC;
 
-    if (flags != CLRDATA_ENUM_MEM_MINI && flags != CLRDATA_ENUM_MEM_TRIAGE)
+    if (flags != CLRDATA_ENUM_MEM_MINI && flags != CLRDATA_ENUM_MEM_TRIAGE && flags != CLRDATA_ENUM_MEM_HEAP2)
     {
         // Modules are enumerated already for minidumps, save the empty calls.
         if (m_module.IsValid())
@@ -2505,7 +2503,7 @@ DebuggerJitInfo::EnumMemoryRegions(CLRDataEnumMemoryFlags flags)
         m_methodInfo->EnumMemoryRegions(flags);
     }
 
-    if (flags != CLRDATA_ENUM_MEM_MINI && flags != CLRDATA_ENUM_MEM_TRIAGE)
+    if (flags != CLRDATA_ENUM_MEM_MINI && flags != CLRDATA_ENUM_MEM_TRIAGE && flags != CLRDATA_ENUM_MEM_HEAP2)
     {
         if (m_nativeCodeVersion.GetMethodDesc().IsValid())
         {

src/coreclr/inc/clrdata.idl

@@ -308,10 +308,12 @@ interface ICLRDataLoggingCallback : IUnknown
 typedef enum CLRDataEnumMemoryFlags
 {
     CLRDATA_ENUM_MEM_DEFAULT = 0x0,
-    CLRDATA_ENUM_MEM_MINI = CLRDATA_ENUM_MEM_DEFAULT,       // generating skinny mini-dump
-    CLRDATA_ENUM_MEM_HEAP = 0x1,        // generating heap dump
-    CLRDATA_ENUM_MEM_TRIAGE = 0x2,      // generating triage mini-dump
-
+    CLRDATA_ENUM_MEM_MINI = CLRDATA_ENUM_MEM_DEFAULT,   // generating skinny mini-dump
+    CLRDATA_ENUM_MEM_HEAP = 0x1,                        // generating heap dump
+    CLRDATA_ENUM_MEM_TRIAGE = 0x2,                      // generating triage mini-dump
+    /* Generate heap dumps faster with less memory usage than CLRDATA_ENUM_MEM_HEAP by adding
+       the loader heaps instead of traversing all the individual runtime data structures. */
+    CLRDATA_ENUM_MEM_HEAP2 = 0x3,
     /* More bits to be added here later */
 } CLRDataEnumMemoryFlags;