Adjust allocation behavior for type system to attempt to reduce TLB c…

src/coreclr/inc/clrconfigvalues.h

@@ -791,6 +791,13 @@ RETAIL_CONFIG_DWORD_INFO(EXTERNAL_EnableArm64Rcpc2,             W("EnableArm64Rc
 RETAIL_CONFIG_DWORD_INFO(EXTERNAL_EnableArm64Sve,               W("EnableArm64Sve"),            1, "Allows Arm64 SVE hardware intrinsics to be disabled")
 #endif
 
+///
+/// TypeSystem
+///
+RETAIL_CONFIG_DWORD_INFO(EXTERNAL_MaxMethodsToContributeToGenericDictionary,               W("MaxMethodsToContributeToGenericDictionary"),            20, "Allows adjusting the heuristic for generic dictionary growth")
+RETAIL_CONFIG_DWORD_INFO(EXTERNAL_MethodsWhichContributeFullyToGenericDictionary,          W("MethodsWhichContributeFullyToGenericDictionary"),       10, "Allows adjusting the heuristic for generic dictionary growth")
+RETAIL_CONFIG_DWORD_INFO(EXTERNAL_UseHighFrequencyMethodTableHeap,                         W("UseHighFrequencyMethodTableHeap"),             1,  "Control whether or not there is a special high frequency method table heap")
+
 ///
 /// Uncategorized
 ///

src/coreclr/vm/appdomain.hpp

@@ -674,12 +674,19 @@ class ThreadStaticHandleTable
 //
 
 #define LOW_FREQUENCY_HEAP_RESERVE_SIZE        (3 * GetOsPageSize())
+#define LOW_FREQUENCY_HEAP_INITIAL_RESERVE_SIZE (512 * GetOsPageSize())
 #define LOW_FREQUENCY_HEAP_COMMIT_SIZE         (1 * GetOsPageSize())
 
 #define HIGH_FREQUENCY_HEAP_RESERVE_SIZE       (10 * GetOsPageSize())
+#define HIGH_FREQUENCY_HEAP_INITIAL_RESERVE_SIZE (768 * GetOsPageSize())
 #define HIGH_FREQUENCY_HEAP_COMMIT_SIZE        (1 * GetOsPageSize())
 
+#define HIGH_FREQUENCY_METHODTABLE_HEAP_RESERVE_SIZE (10 * GetOsPageSize())
+#define HIGH_FREQUENCY_METHODTABLE_HEAP_INITIAL_RESERVE_SIZE (253 * GetOsPageSize())
+#define HIGH_FREQUENCY_METHODTABLE_HEAP_COMMIT_SIZE (1 * GetOsPageSize())
+
 #define STUB_HEAP_RESERVE_SIZE                 (3 * GetOsPageSize())
+#define STUB_HEAP_INITIAL_RESERVE_SIZE         (3 * GetOsPageSize())
 #define STUB_HEAP_COMMIT_SIZE                  (1 * GetOsPageSize())
 
 // --------------------------------------------------------------------------------

src/coreclr/vm/array.cpp

@@ -314,7 +314,7 @@ MethodTable* Module::CreateArrayMethodTable(TypeHandle elemTypeHnd, CorElementTy
 
     // ArrayClass already includes one void*
     LoaderAllocator* pAllocator= this->GetLoaderAllocator();
-    BYTE* pMemory = (BYTE *)pamTracker->Track(pAllocator->GetHighFrequencyHeap()->AllocMem(S_SIZE_T(cbArrayClass) +
+    BYTE* pMemory = (BYTE *)pamTracker->Track(pAllocator->GetHighFrequencyMethodTableHeap()->AllocMem(S_SIZE_T(cbArrayClass) +
                                                                                             S_SIZE_T(cbMT)));
 
     // Note: Memory allocated on loader heap is zero filled
@@ -334,7 +334,7 @@ MethodTable* Module::CreateArrayMethodTable(TypeHandle elemTypeHnd, CorElementTy
     MethodTable* pMT = (MethodTable *) pMTHead;
 
     // Allocate the private data block ("private" during runtime in the ngen'ed case).
-    pMT->AllocateAuxiliaryData(pAllocator, this, pamTracker, false, static_cast<WORD>(numNonVirtualSlots));
+    pMT->AllocateAuxiliaryData(pAllocator->GetHighFrequencyMethodTableHeap(), this, pamTracker, false, static_cast<WORD>(numNonVirtualSlots));
     pMT->SetLoaderAllocator(pAllocator);
     pMT->SetModule(elemTypeHnd.GetModule());
 

src/coreclr/vm/eeconfig.cpp

@@ -772,6 +772,9 @@ HRESULT EEConfig::sync()
 #if defined(FEATURE_GDBJIT_FRAME)
     fGDBJitEmitDebugFrame = CLRConfig::GetConfigValue(CLRConfig::INTERNAL_GDBJitEmitDebugFrame) != 0;
 #endif
+
+    maxMethodsToContributeToGenericDictionary = CLRConfig::GetConfigValue(CLRConfig::EXTERNAL_MaxMethodsToContributeToGenericDictionary);
+    methodsWhichContributeFullyToGenericDictionary = CLRConfig::GetConfigValue(CLRConfig::EXTERNAL_MethodsWhichContributeFullyToGenericDictionary);
     return hr;
 }
 

src/coreclr/vm/eeconfig.h

@@ -448,10 +448,17 @@ class EEConfig
 
 #endif
 
+    uint32_t GetMaxMethodsToContributeToGenericDictionary() const { LIMITED_METHOD_CONTRACT; return maxMethodsToContributeToGenericDictionary; }
+    uint32_t GetMethodsWhichContributeFullyToGenericDictionary() const { LIMITED_METHOD_CONTRACT; return methodsWhichContributeFullyToGenericDictionary; }
+
 private: //----------------------------------------------------------------
 
     bool fInited;                   // have we synced to the registry at least once?
 
+    // Type system config
+    uint32_t maxMethodsToContributeToGenericDictionary;
+    uint32_t methodsWhichContributeFullyToGenericDictionary;
+
     // Jit-config
 
     DWORD dwJitHostMaxSlabCache;       // max size for jit host slab cache

src/coreclr/vm/genericdict.cpp

@@ -107,6 +107,10 @@ DictionaryLayout::GetDictionarySizeFromLayout(
 }
 
 #ifndef DACCESS_COMPILE
+
+static int s_GenericDictMTExpandedSlotsUsed = 0;
+static int s_GenericDictMDExpandedSlotsUsed = 0;
+
 //---------------------------------------------------------------------------------------
 //
 // Find a token in the dictionary layout and return the offsets of indirections
@@ -131,8 +135,9 @@ BOOL DictionaryLayout::FindTokenWorker(LoaderAllocator*                 pAllocat
                                        CORINFO_RUNTIME_LOOKUP*          pResult,
                                        WORD*                            pSlotOut,
                                        DWORD                            scanFromSlot /* = 0 */,
-                                       BOOL                             useEmptySlotIfFound /* = FALSE */)
-
+                                       BOOL                             useEmptySlotIfFound /* = FALSE */,
+                                       MethodTable*                     pMT,
+                                       MethodDesc*                      pMD)
 {
     CONTRACTL
     {
@@ -235,6 +240,21 @@ BOOL DictionaryLayout::FindTokenWorker(LoaderAllocator*                 pAllocat
             // A lock should be taken by FindToken before being allowed to use an empty slot in the layout
             _ASSERT(SystemDomain::SystemModule()->m_DictionaryCrst.OwnedByCurrentThread());
 
+            if (iSlot >= pDictLayout->GetNumInitialSlots())
+            {
+                // Expanded slot used, report this to the log, and gather some statistics
+                if (pMT != NULL)
+                {
+                    s_GenericDictMTExpandedSlotsUsed++;
+                    LOG((LF_CLASSLOADER, LL_INFO100, "GENERICDICT: Used expanded slot %d for type %s, overall expanded slots %d\n", (int)iSlot, pMT->GetDebugClassName(), s_GenericDictMTExpandedSlotsUsed));
+                }
+                else
+                {
+                    s_GenericDictMDExpandedSlotsUsed++;
+                    LOG((LF_CLASSLOADER, LL_INFO100, "GENERICDICT: Used expanded slot %d for method %s::%s, overall expanded slots %d\n", (int)iSlot, pMD->m_pszDebugClassName, pMD->m_pszDebugMethodName, s_GenericDictMDExpandedSlotsUsed));
+                }
+            }
+
             PVOID pResultSignature = pSigBuilder == NULL ? pSig : CreateSignatureWithSlotData(pSigBuilder, pAllocator, slot);
             pDictLayout->m_slots[iSlot].m_signature = pResultSignature;
             pDictLayout->m_slots[iSlot].m_signatureSource = signatureSource;
@@ -334,13 +354,13 @@ BOOL DictionaryLayout::FindToken(MethodTable*                       pMT,
 
     DWORD cbSig = -1;
     pSig = pSigBuilder != NULL ? (BYTE*)pSigBuilder->GetSignature(&cbSig) : pSig;
-    if (FindTokenWorker(pAllocator, pMT->GetNumGenericArgs(), pMT->GetClass()->GetDictionaryLayout(), pSigBuilder, pSig, cbSig, nFirstOffset, signatureSource, pResult, pSlotOut, 0, FALSE))
+    if (FindTokenWorker(pAllocator, pMT->GetNumGenericArgs(), pMT->GetClass()->GetDictionaryLayout(), pSigBuilder, pSig, cbSig, nFirstOffset, signatureSource, pResult, pSlotOut, 0, FALSE, pMT, NULL))
         return TRUE;
 
     CrstHolder ch(&SystemDomain::SystemModule()->m_DictionaryCrst);
     {
         // Try again under lock in case another thread already expanded the dictionaries or filled an empty slot
-        if (FindTokenWorker(pMT->GetLoaderAllocator(), pMT->GetNumGenericArgs(), pMT->GetClass()->GetDictionaryLayout(), pSigBuilder, pSig, cbSig, nFirstOffset, signatureSource, pResult, pSlotOut, *pSlotOut, TRUE))
+        if (FindTokenWorker(pMT->GetLoaderAllocator(), pMT->GetNumGenericArgs(), pMT->GetClass()->GetDictionaryLayout(), pSigBuilder, pSig, cbSig, nFirstOffset, signatureSource, pResult, pSlotOut, *pSlotOut, TRUE, pMT, NULL))
             return TRUE;
 
         DictionaryLayout* pOldLayout = pMT->GetClass()->GetDictionaryLayout();
@@ -381,13 +401,13 @@ BOOL DictionaryLayout::FindToken(MethodDesc*                        pMD,
 
     DWORD cbSig = -1;
     pSig = pSigBuilder != NULL ? (BYTE*)pSigBuilder->GetSignature(&cbSig) : pSig;
-    if (FindTokenWorker(pAllocator, pMD->GetNumGenericMethodArgs(), pMD->GetDictionaryLayout(), pSigBuilder, pSig, cbSig, nFirstOffset, signatureSource, pResult, pSlotOut, 0, FALSE))
+    if (FindTokenWorker(pAllocator, pMD->GetNumGenericMethodArgs(), pMD->GetDictionaryLayout(), pSigBuilder, pSig, cbSig, nFirstOffset, signatureSource, pResult, pSlotOut, 0, FALSE, NULL, pMD))
         return TRUE;
 
     CrstHolder ch(&SystemDomain::SystemModule()->m_DictionaryCrst);
     {
         // Try again under lock in case another thread already expanded the dictionaries or filled an empty slot
-        if (FindTokenWorker(pAllocator, pMD->GetNumGenericMethodArgs(), pMD->GetDictionaryLayout(), pSigBuilder, pSig, cbSig, nFirstOffset, signatureSource, pResult, pSlotOut, *pSlotOut, TRUE))
+        if (FindTokenWorker(pAllocator, pMD->GetNumGenericMethodArgs(), pMD->GetDictionaryLayout(), pSigBuilder, pSig, cbSig, nFirstOffset, signatureSource, pResult, pSlotOut, *pSlotOut, TRUE, NULL, pMD))
             return TRUE;
 
         DictionaryLayout* pOldLayout = pMD->GetDictionaryLayout();

src/coreclr/vm/genericdict.h

@@ -120,7 +120,10 @@ class DictionaryLayout
                                 CORINFO_RUNTIME_LOOKUP*             pResult,
                                 WORD*                               pSlotOut,
                                 DWORD                               scanFromSlot,
-                                BOOL                                useEmptySlotIfFound);
+                                BOOL                                useEmptySlotIfFound,
+                                MethodTable*                        pMT,
+                                MethodDesc*                         pMD
+                                );
 
 
     static DictionaryLayout* ExpandDictionaryLayout(LoaderAllocator*                pAllocator,

src/coreclr/vm/generics.cpp

@@ -269,7 +269,22 @@ ClassLoader::CreateTypeHandleForNonCanonicalGenericInstantiation(
         ThrowHR(COR_E_OVERFLOW);
     }
 
-    BYTE* pMemory = (BYTE *) pamTracker->Track(pAllocator->GetHighFrequencyHeap()->AllocMem( allocSize ));
+    // Types with instantiations containing generic variables, or interfaces are less likely to be highly active.
+    LoaderHeap *pLoaderHeapToUse;
+    if (fContainsGenericVariables)
+    {
+        pLoaderHeapToUse = pAllocator->GetLowFrequencyHeap();
+    }
+    else if (pOldMT->IsInterface())
+    {
+        pLoaderHeapToUse = pAllocator->GetHighFrequencyHeap();
+    }
+    else
+    {
+        pLoaderHeapToUse = pAllocator->GetHighFrequencyMethodTableHeap();
+    }
+
+    BYTE* pMemory = (BYTE *) pamTracker->Track(pLoaderHeapToUse->AllocMem( allocSize ));
 
     // Head of MethodTable memory
     MethodTable *pMT = (MethodTable*) (pMemory + cbGC);
@@ -278,7 +293,7 @@ ClassLoader::CreateTypeHandleForNonCanonicalGenericInstantiation(
     memcpy((BYTE*)pMT - cbGC, (BYTE*) pOldMT - cbGC, cbGC);
 
     // Allocate the private data block
-    pMT->AllocateAuxiliaryData(pAllocator, pLoaderModule, pamTracker, fHasGenericsStaticsInfo);
+    pMT->AllocateAuxiliaryData(pLoaderHeapToUse, pLoaderModule, pamTracker, fHasGenericsStaticsInfo);
     pMT->SetModule(pOldMT->GetModule());
 
     pMT->GetAuxiliaryDataForWrite()->SetIsNotFullyLoadedForBuildMethodTable();

src/coreclr/vm/loaderallocator.cpp

@@ -27,6 +27,7 @@ LoaderAllocator::LoaderAllocator(bool collectible) :
     m_InitialReservedMemForLoaderHeaps = NULL;
     m_pLowFrequencyHeap = NULL;
     m_pHighFrequencyHeap = NULL;
+    m_pHighFrequencyMethodTableHeap = NULL;
     m_pStubHeap = NULL;
     m_pPrecodeHeap = NULL;
     m_pExecutableHeap = NULL;
@@ -1078,12 +1079,14 @@ void LoaderAllocator::Init(BaseDomain *pDomain, BYTE *pExecutableHeapMemory)
 
     DWORD dwLowFrequencyHeapReserveSize;
     DWORD dwHighFrequencyHeapReserveSize;
+    DWORD dwHighFrequencyMethodTableHeapReserveSize;
     DWORD dwStubHeapReserveSize;
     DWORD dwExecutableHeapReserveSize;
     DWORD dwCodeHeapReserveSize;
     DWORD dwVSDHeapReserveSize;
 
     dwExecutableHeapReserveSize = 0;
+    bool disableHighFrequencyMethodTableHeap = false;
 
     if (IsCollectible())
     {
@@ -1092,12 +1095,21 @@ void LoaderAllocator::Init(BaseDomain *pDomain, BYTE *pExecutableHeapMemory)
         dwStubHeapReserveSize          = COLLECTIBLE_STUB_HEAP_SIZE;
         dwCodeHeapReserveSize          = COLLECTIBLE_CODEHEAP_SIZE;
         dwVSDHeapReserveSize           = COLLECTIBLE_VIRTUALSTUBDISPATCH_HEAP_SPACE;
+        dwHighFrequencyMethodTableHeapReserveSize = 0;
     }
     else
     {
-        dwLowFrequencyHeapReserveSize  = LOW_FREQUENCY_HEAP_RESERVE_SIZE;
-        dwHighFrequencyHeapReserveSize = HIGH_FREQUENCY_HEAP_RESERVE_SIZE;
-        dwStubHeapReserveSize          = STUB_HEAP_RESERVE_SIZE;
+        dwLowFrequencyHeapReserveSize  = LOW_FREQUENCY_HEAP_INITIAL_RESERVE_SIZE;
+        dwHighFrequencyHeapReserveSize = HIGH_FREQUENCY_HEAP_INITIAL_RESERVE_SIZE;
+        dwStubHeapReserveSize          = STUB_HEAP_INITIAL_RESERVE_SIZE;
+        dwHighFrequencyMethodTableHeapReserveSize = HIGH_FREQUENCY_METHODTABLE_HEAP_INITIAL_RESERVE_SIZE - dwStubHeapReserveSize;
+
+        disableHighFrequencyMethodTableHeap = (CLRConfig::GetConfigValue(CLRConfig::EXTERNAL_UseHighFrequencyMethodTableHeap) == 0);
+        if (disableHighFrequencyMethodTableHeap)
+        {
+            dwHighFrequencyHeapReserveSize += dwHighFrequencyMethodTableHeapReserveSize;
+            dwHighFrequencyMethodTableHeapReserveSize = 0;
+        }
 
         // Non-collectible assemblies do not reserve space for these heaps.
         dwCodeHeapReserveSize = 0;
@@ -1119,7 +1131,8 @@ void LoaderAllocator::Init(BaseDomain *pDomain, BYTE *pExecutableHeapMemory)
                                 + dwStubHeapReserveSize
                                 + dwCodeHeapReserveSize
                                 + dwVSDHeapReserveSize
-                                + dwExecutableHeapReserveSize;
+                                + dwExecutableHeapReserveSize
+                                + dwHighFrequencyMethodTableHeapReserveSize;
 
     dwTotalReserveMemSize = (DWORD) ALIGN_UP(dwTotalReserveMemSize, VIRTUAL_ALLOC_RESERVE_GRANULARITY);
 
@@ -1175,7 +1188,25 @@ void LoaderAllocator::Init(BaseDomain *pDomain, BYTE *pExecutableHeapMemory)
     initReservedMem += dwHighFrequencyHeapReserveSize;
 
     if (IsCollectible())
+    {
         m_pLowFrequencyHeap = m_pHighFrequencyHeap;
+        m_pHighFrequencyMethodTableHeap = m_pHighFrequencyHeap;
+    }
+    else
+    {
+        if (disableHighFrequencyMethodTableHeap)
+        {
+            m_pHighFrequencyMethodTableHeap = m_pHighFrequencyHeap;
+        }
+        else
+        {
+            m_pHighFrequencyMethodTableHeap = new (&m_HighFreqMethodTableHeapInstance) LoaderHeap(HIGH_FREQUENCY_METHODTABLE_HEAP_RESERVE_SIZE,
+                                                                                                  HIGH_FREQUENCY_METHODTABLE_HEAP_COMMIT_SIZE,
+                                                                                                  initReservedMem,
+                                                                                                  dwHighFrequencyMethodTableHeapReserveSize);
+            initReservedMem += dwHighFrequencyMethodTableHeapReserveSize;
+        }
+    }
 
 #if defined(_DEBUG) && defined(STUBLINKER_GENERATES_UNWIND_INFO)
     m_pHighFrequencyHeap->m_fPermitStubsWithUnwindInfo = TRUE;

src/coreclr/vm/loaderallocator.hpp

@@ -302,12 +302,14 @@ class LoaderAllocator
     BYTE *              m_InitialReservedMemForLoaderHeaps;
     BYTE                m_LowFreqHeapInstance[sizeof(LoaderHeap)];
     BYTE                m_HighFreqHeapInstance[sizeof(LoaderHeap)];
+    BYTE                m_HighFreqMethodTableHeapInstance[sizeof(LoaderHeap)];
     BYTE                m_StubHeapInstance[sizeof(LoaderHeap)];
     BYTE                m_PrecodeHeapInstance[sizeof(CodeFragmentHeap)];
     BYTE                m_FixupPrecodeHeapInstance[sizeof(LoaderHeap)];
     BYTE                m_NewStubPrecodeHeapInstance[sizeof(LoaderHeap)];
     PTR_LoaderHeap      m_pLowFrequencyHeap;
     PTR_LoaderHeap      m_pHighFrequencyHeap;
+    PTR_LoaderHeap      m_pHighFrequencyMethodTableHeap;
     PTR_LoaderHeap      m_pStubHeap; // stubs for PInvoke, remoting, etc
     PTR_CodeFragmentHeap m_pPrecodeHeap;
     PTR_LoaderHeap      m_pExecutableHeap;
@@ -582,6 +584,12 @@ class LoaderAllocator
         return m_pHighFrequencyHeap;
     }
 
+    PTR_LoaderHeap GetHighFrequencyMethodTableHeap()
+    {
+        LIMITED_METHOD_CONTRACT;
+        return m_pHighFrequencyMethodTableHeap;
+    }
+
     PTR_LoaderHeap GetStubHeap()
     {
         LIMITED_METHOD_CONTRACT;

src/coreclr/vm/methodtable.cpp

@@ -671,7 +671,7 @@ MethodDesc *MethodTable::GetMethodDescForComInterfaceMethod(MethodDesc *pItfMD,
 }
 #endif // FEATURE_COMINTEROP
 
-void MethodTable::AllocateAuxiliaryData(LoaderAllocator *pAllocator, Module *pLoaderModule, AllocMemTracker *pamTracker, bool hasGenericStatics, WORD nonVirtualSlots, S_SIZE_T extraAllocation)
+void MethodTable::AllocateAuxiliaryData(LoaderHeap *pHeap, Module *pLoaderModule, AllocMemTracker *pamTracker, bool hasGenericStatics, WORD nonVirtualSlots, S_SIZE_T extraAllocation)
 {
     S_SIZE_T cbAuxiliaryData = S_SIZE_T(sizeof(MethodTableAuxiliaryData));
 
@@ -687,7 +687,7 @@ void MethodTable::AllocateAuxiliaryData(LoaderAllocator *pAllocator, Module *pLo
         ThrowHR(COR_E_OVERFLOW);
 
     BYTE* pAuxiliaryDataRegion = (BYTE *)
-        pamTracker->Track(pAllocator->GetHighFrequencyHeap()->AllocMem(cbAuxiliaryData));
+        pamTracker->Track(pHeap->AllocMem(cbAuxiliaryData));
 
     MethodTableAuxiliaryData * pMTAuxiliaryData;
     pMTAuxiliaryData = (MethodTableAuxiliaryData *)(pAuxiliaryDataRegion + prependedAllocationSpace);
@@ -724,7 +724,7 @@ MethodTable* CreateMinimalMethodTable(Module* pContainingModule,
     // memset(pMT, 0, sizeof(MethodTable));
 
     // Allocate the private data block ("private" during runtime in the ngen'ed case).
-    pMT->AllocateAuxiliaryData(pLoaderAllocator, pContainingModule, pamTracker);
+    pMT->AllocateAuxiliaryData(pLoaderAllocator->GetHighFrequencyHeap(), pContainingModule, pamTracker);
     pMT->SetModule(pContainingModule);
     pMT->SetLoaderAllocator(pLoaderAllocator);
 

src/coreclr/vm/methodtable.h

@@ -2685,7 +2685,7 @@ class MethodTable
     // ------------------------------------------------------------------
 
 #ifndef DACCESS_COMPILE
-    void AllocateAuxiliaryData(LoaderAllocator *pAllocator, Module *pLoaderModule, AllocMemTracker *pamTracker, bool hasGenericStatics = false, WORD nonVirtualSlots = 0, S_SIZE_T extraAllocation = S_SIZE_T(0));
+    void AllocateAuxiliaryData(LoaderHeap *pHeap, Module *pLoaderModule, AllocMemTracker *pamTracker, bool hasGenericStatics = false, WORD nonVirtualSlots = 0, S_SIZE_T extraAllocation = S_SIZE_T(0));
 #endif
 
     inline PTR_Const_MethodTableAuxiliaryData GetAuxiliaryData() const