win-x86: Replace "ret n" thunks with assembly code (dotnet#115089)

Replace x86 COM "ret n" thunks with assembly code that emulates
"ret <n>" for an "<n>" specified using a variable value stored
in register.

Author: filipnavara

src/coreclr/inc/CrstTypes.def

@@ -208,10 +208,6 @@ Crst MethodTableExposedObject
     Unordered
 End
 
-Crst RetThunkCache
-    AcquiredBefore LoaderHeap
-End
-
 Crst FuncPtrStubs
     AcquiredBefore IbcProfile LoaderHeap UniqueStack CodeFragmentHeap JumpStubCache
 End

src/coreclr/inc/crsttypes_generated.h

@@ -96,30 +96,29 @@ enum CrstType
     CrstReadyToRunEntryPointToMethodDescMap = 78,
     CrstReflection = 79,
     CrstReJITGlobalRequest = 80,
-    CrstRetThunkCache = 81,
-    CrstSigConvert = 82,
-    CrstSingleUseLock = 83,
-    CrstStressLog = 84,
-    CrstStubCache = 85,
-    CrstStubDispatchCache = 86,
-    CrstSyncBlockCache = 87,
-    CrstSyncHashLock = 88,
-    CrstSystemDomain = 89,
-    CrstSystemDomainDelayedUnloadList = 90,
-    CrstThreadIdDispenser = 91,
-    CrstThreadLocalStorageLock = 92,
-    CrstThreadStore = 93,
-    CrstTieredCompilation = 94,
-    CrstTypeEquivalenceMap = 95,
-    CrstTypeIDMap = 96,
-    CrstUMEntryThunkCache = 97,
-    CrstUMEntryThunkFreeListLock = 98,
-    CrstUniqueStack = 99,
-    CrstUnresolvedClassLock = 100,
-    CrstUnwindInfoTableLock = 101,
-    CrstVSDIndirectionCellLock = 102,
-    CrstWrapperTemplate = 103,
-    kNumberOfCrstTypes = 104
+    CrstSigConvert = 81,
+    CrstSingleUseLock = 82,
+    CrstStressLog = 83,
+    CrstStubCache = 84,
+    CrstStubDispatchCache = 85,
+    CrstSyncBlockCache = 86,
+    CrstSyncHashLock = 87,
+    CrstSystemDomain = 88,
+    CrstSystemDomainDelayedUnloadList = 89,
+    CrstThreadIdDispenser = 90,
+    CrstThreadLocalStorageLock = 91,
+    CrstThreadStore = 92,
+    CrstTieredCompilation = 93,
+    CrstTypeEquivalenceMap = 94,
+    CrstTypeIDMap = 95,
+    CrstUMEntryThunkCache = 96,
+    CrstUMEntryThunkFreeListLock = 97,
+    CrstUniqueStack = 98,
+    CrstUnresolvedClassLock = 99,
+    CrstUnwindInfoTableLock = 100,
+    CrstVSDIndirectionCellLock = 101,
+    CrstWrapperTemplate = 102,
+    kNumberOfCrstTypes = 103
 };
 
 #endif // __CRST_TYPES_INCLUDED
@@ -211,7 +210,6 @@ int g_rgCrstLevelMap[] =
     9,          // CrstReadyToRunEntryPointToMethodDescMap
     7,          // CrstReflection
     15,         // CrstReJITGlobalRequest
-    3,          // CrstRetThunkCache
     3,          // CrstSigConvert
     4,          // CrstSingleUseLock
     -1,         // CrstStressLog
@@ -320,7 +318,6 @@ LPCSTR g_rgCrstNameMap[] =
     "CrstReadyToRunEntryPointToMethodDescMap",
     "CrstReflection",
     "CrstReJITGlobalRequest",
-    "CrstRetThunkCache",
     "CrstSigConvert",
     "CrstSingleUseLock",
     "CrstStressLog",

src/coreclr/vm/clrtocomcall.cpp

@@ -950,76 +950,3 @@ BOOL CLRToCOMMethodFrame::TraceFrame_Impl(Thread *thread, BOOL fromPatch,
 
     return TRUE;
 }
-
-#ifdef TARGET_X86
-
-#ifndef DACCESS_COMPILE
-
-CrstStatic   CLRToCOMCall::s_RetThunkCacheCrst;
-SHash<CLRToCOMCall::RetThunkSHashTraits> *CLRToCOMCall::s_pRetThunkCache = NULL;
-
-// One time init.
-void CLRToCOMCall::Init()
-{
-    CONTRACTL
-    {
-        THROWS;
-        GC_NOTRIGGER;
-        MODE_ANY;
-    }
-    CONTRACTL_END;
-
-    s_RetThunkCacheCrst.Init(CrstRetThunkCache);
-}
-
-LPVOID CLRToCOMCall::GetRetThunk(UINT numStackBytes)
-{
-    STANDARD_VM_CONTRACT;
-
-    LPVOID pRetThunk = NULL;
-    CrstHolder crst(&s_RetThunkCacheCrst);
-
-    // Lazily allocate the ret thunk cache.
-    if (s_pRetThunkCache == NULL)
-        s_pRetThunkCache = new SHash<RetThunkSHashTraits>();
-
-    const RetThunkCacheElement *pElement = s_pRetThunkCache->LookupPtr(numStackBytes);
-    if (pElement != NULL)
-    {
-        pRetThunk = pElement->m_pRetThunk;
-    }
-    else
-    {
-        // cache miss -> create a new thunk
-        AllocMemTracker dummyAmTracker;
-        size_t thunkSize = (numStackBytes == 0) ? 1 : 3;
-        pRetThunk = (LPVOID)dummyAmTracker.Track(SystemDomain::GetGlobalLoaderAllocator()->GetExecutableHeap()->AllocMem(S_SIZE_T(thunkSize)));
-
-        ExecutableWriterHolder<BYTE> thunkWriterHolder((BYTE *)pRetThunk, thunkSize);
-        BYTE *pThunkRW = thunkWriterHolder.GetRW();
-
-        if (numStackBytes == 0)
-        {
-            pThunkRW[0] = 0xc3;
-        }
-        else
-        {
-            pThunkRW[0] = 0xc2;
-            *(USHORT *)&pThunkRW[1] = (USHORT)numStackBytes;
-        }
-
-        // add it to the cache
-        RetThunkCacheElement element;
-        element.m_cbStack = numStackBytes;
-        element.m_pRetThunk = pRetThunk;
-        s_pRetThunkCache->Add(element);
-
-        dummyAmTracker.SuppressRelease();
-    }
-
-    return pRetThunk;
-}
-
-#endif // !DACCESS_COMPILE
-
-#endif // TARGET_X86

src/coreclr/vm/clrtocomcall.h

@@ -30,42 +30,8 @@ class CLRToCOMCall
         static PCODE       GetStubForILStub(MethodDesc* pMD, MethodDesc** ppStubMD);
 
         static CLRToCOMCallInfo *PopulateCLRToCOMCallMethodDesc(MethodDesc* pMD, DWORD* pdwStubFlags);
-
-#ifdef TARGET_X86
-        static void Init();
-        static LPVOID GetRetThunk(UINT numStackBytes);
-#endif // TARGET_X86
     private:
         CLRToCOMCall();     // prevent "new"'s on this class
-
-#ifdef TARGET_X86
-    struct RetThunkCacheElement
-    {
-        RetThunkCacheElement()
-        {
-            LIMITED_METHOD_CONTRACT;
-            m_cbStack = 0;
-            m_pRetThunk = NULL;
-        }
-
-        UINT m_cbStack;
-        LPVOID m_pRetThunk;
-    };
-
-    class RetThunkSHashTraits : public NoRemoveSHashTraits< DefaultSHashTraits<RetThunkCacheElement> >
-    {
-    public:
-        typedef UINT key_t;
-        static key_t GetKey(element_t e)       { LIMITED_METHOD_CONTRACT; return e.m_cbStack; }
-        static BOOL Equals(key_t k1, key_t k2) { LIMITED_METHOD_CONTRACT; return (k1 == k2); }
-        static count_t Hash(key_t k)           { LIMITED_METHOD_CONTRACT; return (count_t)(size_t)k; }
-        static const element_t Null()          { LIMITED_METHOD_CONTRACT; return RetThunkCacheElement(); }
-        static bool IsNull(const element_t &e) { LIMITED_METHOD_CONTRACT; return (e.m_pRetThunk == NULL); }
-    };
-
-    static SHash<RetThunkSHashTraits> *s_pRetThunkCache;
-    static CrstStatic s_RetThunkCacheCrst;
-#endif // TARGET_X86
 };
 
 #endif // __CLRTOCOMCALL_H__

src/coreclr/vm/dllimport.cpp

@@ -5783,9 +5783,11 @@ void CreateCLRToDispatchCOMStub(
                             iLCIDArg);
 
     _ASSERTE(pMD->IsCLRToCOMCall()); // no generic disp-calls
-    ((CLRToCOMCallMethodDesc *)pMD)->InitRetThunk();
-}
 
+#ifdef TARGET_X86
+    ((CLRToCOMCallMethodDesc *)pMD)->InitStackPop();
+#endif
+}
 
 #endif // FEATURE_COMINTEROP
 

src/coreclr/vm/i386/asmconstants.h

@@ -227,8 +227,8 @@ ASMCONSTANTS_C_ASSERT(OFFSETOF__FrameHandlerExRecord__m_pEntryFrame == offsetof(
 #define CLRToCOMCallMethodDesc__m_pCLRToCOMCallInfo DBG_FRE(0x20, 0xC)
 ASMCONSTANTS_C_ASSERT(CLRToCOMCallMethodDesc__m_pCLRToCOMCallInfo == offsetof(CLRToCOMCallMethodDesc, m_pCLRToCOMCallInfo))
 
-#define CLRToCOMCallInfo__m_pRetThunk 0x10
-ASMCONSTANTS_C_ASSERT(CLRToCOMCallInfo__m_pRetThunk == offsetof(CLRToCOMCallInfo, m_pRetThunk))
+#define CLRToCOMCallInfo__m_cbStackPop 0x0e
+ASMCONSTANTS_C_ASSERT(CLRToCOMCallInfo__m_cbStackPop == offsetof(CLRToCOMCallInfo, m_cbStackPop))
 
 #define COMMETHOD_PREPAD_ASM  8
 ASMCONSTANTS_C_ASSERT(COMMETHOD_PREPAD_ASM == COMMETHOD_PREPAD)

src/coreclr/vm/i386/asmhelpers.asm

@@ -1291,14 +1291,19 @@ _GenericCLRToCOMCallStub@0 proc public
 
     ; Get pCLRToCOMCallInfo for return thunk
     mov         ecx, [ebx + CLRToCOMCallMethodDesc__m_pCLRToCOMCallInfo]
+    ; Get size of arguments to pop
+    movzx       ecx, word ptr [ecx + CLRToCOMCallInfo__m_cbStackPop]
+    ; Get the return address, pushed registers on stack are 24 bytes big
+    mov         ebx, [esp + 24]
+    ; Set the return address on stack at the last stack slot
+    mov         [esp + ecx + 24], ebx
 
     STUB_EPILOG_RETURN
 
-    ; Tailcall return thunk
-    jmp [ecx + CLRToCOMCallInfo__m_pRetThunk]
+    ; Move esp to point to the last stack slot where we put the return
+    ; address earlier
+    lea         esp, [esp + ecx]
 
-    ; This will never be executed. It is just to help out stack-walking logic
-    ; which disassembles the epilog to unwind the stack.
     ret
 
 _GenericCLRToCOMCallStub@0 endp

src/coreclr/vm/interoputil.cpp

@@ -3681,9 +3681,6 @@ void InitializeComInterop()
 
     InitializeSListHead(&RCW::s_RCWStandbyList);
     ComCall::Init();
-#ifdef TARGET_X86
-    CLRToCOMCall::Init();
-#endif
     CtxEntryCache::Init();
     ComCallWrapperTemplate::Init();
 #ifdef _DEBUG

src/coreclr/vm/method.cpp

@@ -3964,26 +3964,6 @@ PrecodeType MethodDesc::GetPrecodeType()
 
 #endif // !DACCESS_COMPILE
 
-#ifdef FEATURE_COMINTEROP
-#ifndef DACCESS_COMPILE
-void CLRToCOMCallMethodDesc::InitRetThunk()
-{
-    WRAPPER_NO_CONTRACT;
-
-#ifdef TARGET_X86
-    if (m_pCLRToCOMCallInfo->m_pRetThunk != NULL)
-        return;
-
-    UINT numStackBytes = CbStackPop();
-
-    LPVOID pRetThunk = CLRToCOMCall::GetRetThunk(numStackBytes);
-
-    InterlockedCompareExchangeT<void *>(&m_pCLRToCOMCallInfo->m_pRetThunk, pRetThunk, NULL);
-#endif // TARGET_X86
-}
-#endif //!DACCESS_COMPILE
-#endif // FEATURE_COMINTEROP
-
 #ifndef DACCESS_COMPILE
 void MethodDesc::PrepareForUseAsADependencyOfANativeImageWorker()
 {

src/coreclr/vm/method.hpp

@@ -3342,6 +3342,8 @@ struct CLRToCOMCallInfo
     // Size of outgoing arguments (on stack). This is currently used only
     // on x86 when we have an InlinedCallFrame representing a CLR->COM call.
     WORD        m_cbStackArgumentSize;
+    // Size of incoming arguments (on stack).
+    WORD        m_cbStackPop;
 
     void InitStackArgumentSize()
     {
@@ -3369,9 +3371,6 @@ struct CLRToCOMCallInfo
         _ASSERTE(m_cbStackArgumentSize != 0xFFFF);
         return m_cbStackArgumentSize;
     }
-
-    LPVOID      m_pRetThunk;
-
 #else // TARGET_X86
     void InitStackArgumentSize()
     {
@@ -3396,7 +3395,6 @@ class CLRToCOMCallMethodDesc : public MethodDesc
 public:
     CLRToCOMCallInfo *m_pCLRToCOMCallInfo; // initialized in code:CLRToCOMCall.PopulateCLRToCOMCallMethodDesc
 
-    void InitRetThunk();
     void InitComEventCallInfo();
 
     PCODE * GetAddrOfILStubField()
@@ -3446,6 +3444,12 @@ class CLRToCOMCallMethodDesc : public MethodDesc
         LIMITED_METHOD_CONTRACT;
         m_pCLRToCOMCallInfo->SetStackArgumentSize(cbDstBuffer);
     }
+
+    void InitStackPop()
+    {
+        LIMITED_METHOD_CONTRACT;
+        m_pCLRToCOMCallInfo->m_cbStackPop = (WORD)CbStackPop();
+    }
 #else // TARGET_X86
     void SetStackArgumentSize(WORD cbDstBuffer)
     {