[clr-interp] Add support for unmanaged thiscall calling convention

src/coreclr/vm/callconvbuilder.cpp

@@ -354,8 +354,35 @@ HRESULT CallConv::TryGetUnmanagedCallingConventionFromModOpt(
     }
     IfFailRet(sigPtr.GetData(NULL)); // arg count
 
+#ifdef DEBUG
     PCCOR_SIGNATURE pWalk = sigPtr.GetPtr();
     _ASSERTE(pWalk <= pSig + cSig);
+#endif
+
+    return TryGetUnmanagedCallingConventionFromModOptSigStartingAtRetType(
+        pModule,
+        sigPtr,
+        builder,
+        errorResID);
+}
+
+HRESULT CallConv::TryGetUnmanagedCallingConventionFromModOptSigStartingAtRetType(
+    _In_ CORINFO_MODULE_HANDLE pModule,
+    _In_ SigPointer sig,
+    _Inout_ CallConvBuilder* builder,
+    _Out_ UINT* errorResID)
+{
+    CONTRACTL
+    {
+        STANDARD_VM_CHECK;
+        PRECONDITION(builder != NULL);
+        PRECONDITION(errorResID != NULL);
+    }
+    CONTRACTL_END;
+    PCCOR_SIGNATURE pSig;
+    uint32_t cSig;
+    sig.GetSignature(&pSig, &cSig);
+    PCCOR_SIGNATURE pWalk = pSig;
 
     CallConvBuilder& callConvBuilder = *builder;
     while ((pWalk < (pSig + cSig)) && ((*pWalk == ELEMENT_TYPE_CMOD_OPT) || (*pWalk == ELEMENT_TYPE_CMOD_REQD) || (*pWalk == ELEMENT_TYPE_CMOD_INTERNAL)))

src/coreclr/vm/callconvbuilder.hpp

@@ -80,6 +80,22 @@ namespace CallConv
         _Inout_ CallConvBuilder *builder,
         _Out_ UINT* errorResID);
 
+    //-------------------------------------------------------------------------
+    // Gets the unmanaged calling convention by reading any modopts, starting the sig
+    // walk at the return type in the signature
+    //
+    // Returns:
+    //   S_OK - No errors
+    //   COR_E_BADIMAGEFORMAT - Signature had an invalid format
+    //   COR_E_INVALIDPROGRAM - Program is considered invalid (more
+    //                          than one calling convention specified)
+    //-------------------------------------------------------------------------
+    HRESULT TryGetUnmanagedCallingConventionFromModOptSigStartingAtRetType(
+        _In_ CORINFO_MODULE_HANDLE pModule,
+        _In_ SigPointer sig,
+        _Inout_ CallConvBuilder* builder,
+        _Out_ UINT* errorResID);
+
     //-------------------------------------------------------------------------
     // Gets the calling convention from the UnmanagedCallConv attribute
     //

src/coreclr/vm/callstubgenerator.cpp

@@ -4,7 +4,9 @@
 #if defined(FEATURE_INTERPRETER) && !defined(TARGET_WASM)
 
 #include "callstubgenerator.h"
+#include "callconvbuilder.hpp"
 #include "ecall.h"
+#include "dllimport.h"
 
 extern "C" void InjectInterpStackAlign();
 extern "C" void Load_Stack();
@@ -2267,7 +2269,7 @@ CallStubHeader *CallStubGenerator::GenerateCallStub(MethodDesc *pMD, AllocMemTra
     PCODE *pRoutines = (PCODE*)alloca(tempStorageSize);
     memset(pRoutines, 0, tempStorageSize);
 
-    ComputeCallStub(sig, pRoutines);
+    ComputeCallStub(sig, pRoutines, pMD);
 
     LoaderAllocator *pLoaderAllocator = pMD->GetLoaderAllocator();
     S_SIZE_T finalStubSize(sizeof(CallStubHeader) + m_routineIndex * sizeof(PCODE));
@@ -2364,7 +2366,7 @@ CallStubHeader *CallStubGenerator::GenerateCallStubForSig(MetaSig &sig)
 
     m_interpreterToNative = true; // We always generate the interpreter to native call stub here
 
-    ComputeCallStub(sig, pRoutines);
+    ComputeCallStub(sig, pRoutines, NULL);
 
     xxHash hashState;
     for (int i = 0; i < m_routineIndex; i++)
@@ -2451,8 +2453,179 @@ void CallStubGenerator::TerminateCurrentRoutineIfNotOfNewType(RoutineType type,
     return;
 }
 
-void CallStubGenerator::ComputeCallStub(MetaSig &sig, PCODE *pRoutines)
+//---------------------------------------------------------------------------
+// isNativePrimitiveStructType:
+//    Check if the given struct type is an intrinsic type that should be treated as though
+//    it is not a struct at the unmanaged ABI boundary.
+//
+// Arguments:
+//    pMT - the handle for the struct type.
+//
+// Return Value:
+//    true if the given struct type should be treated as a primitive for unmanaged calls,
+//    false otherwise.
+//
+bool isNativePrimitiveStructType(MethodTable* pMT)
 {
+    if (!pMT->IsIntrinsicType())
+    {
+        return false;
+    }
+    const char* namespaceName = nullptr;
+    const char* typeName      = pMT->GetFullyQualifiedNameInfo(&namespaceName);
+
+    if ((namespaceName == NULL) || (typeName == NULL))
+    {
+        return false;
+    }
+
+    if (strcmp(namespaceName, "System.Runtime.InteropServices") != 0)
+    {
+        return false;
+    }
+
+    return strcmp(typeName, "CLong") == 0 || strcmp(typeName, "CULong") == 0 || strcmp(typeName, "NFloat") == 0;
+}
+
+void CallStubGenerator::ComputeCallStub(MetaSig &sig, PCODE *pRoutines, MethodDesc *pMD)
+{
+    bool rewriteMetaSigFromExplicitThisToHasThis = false;
+    bool unmanagedThisCallConv = false;
+
+    bool hasUnmanagedCallConv = false;
+    CorInfoCallConvExtension unmanagedCallConv = CorInfoCallConvExtension::C;
+
+    if (pMD != NULL && (pMD->IsPInvoke()))
+    {
+        PInvoke::GetCallingConvention_IgnoreErrors(pMD, &unmanagedCallConv, NULL);
+        hasUnmanagedCallConv = true;
+    }
+    else if (pMD != NULL && pMD->HasUnmanagedCallersOnlyAttribute())
+    {
+        if (CallConv::TryGetCallingConventionFromUnmanagedCallersOnly(pMD, &unmanagedCallConv))
+        {
+            if (sig.GetCallingConvention() == IMAGE_CEE_CS_CALLCONV_VARARG)
+            {
+                unmanagedCallConv = CorInfoCallConvExtension::C;
+            }
+        }
+        else
+        {
+            unmanagedCallConv = CallConv::GetDefaultUnmanagedCallingConvention();
+        }
+        hasUnmanagedCallConv = true;
+    }
+    else
+    {
+        switch (sig.GetCallingConvention())
+        {
+            case IMAGE_CEE_CS_CALLCONV_THISCALL:
+                unmanagedCallConv = CorInfoCallConvExtension::Thiscall;
+                hasUnmanagedCallConv = true;
+                break;
+            case IMAGE_CEE_CS_CALLCONV_UNMANAGED:
+                unmanagedCallConv = GetUnmanagedCallConvExtension(&sig);
+                hasUnmanagedCallConv = true;
+                break;
+        }
+    }
+
+    if (hasUnmanagedCallConv)
+    {
+        switch (unmanagedCallConv)
+        {
+            case CorInfoCallConvExtension::Thiscall:
+            case CorInfoCallConvExtension::CMemberFunction:
+            case CorInfoCallConvExtension::StdcallMemberFunction:
+            case CorInfoCallConvExtension::FastcallMemberFunction:
+                unmanagedThisCallConv = true;
+                break;
+            default:
+                break;
+        }
+    }
+
+#if defined(TARGET_WINDOWS)
+    // On these platforms, when making a ThisCall, or other call using a C++ MemberFunction calling convention,
+    // the "this" pointer is passed in the first argument slot.
+    bool rewriteReturnTypeToForceRetBuf = false;
+    if (unmanagedThisCallConv)
+    {
+        rewriteMetaSigFromExplicitThisToHasThis = true;
+        // Also, any struct type other than a few special cases is returned via return buffer for unmanaged calls
+        CorElementType retType = sig.GetReturnType();
+        sig.Reset();
+
+        if (retType == ELEMENT_TYPE_VALUETYPE)
+        {
+            TypeHandle thRetType = sig.GetRetTypeHandleThrowing();
+            MethodTable* pMTRetType = thRetType.AsMethodTable();
+
+            if (pMTRetType->GetInternalCorElementType() == ELEMENT_TYPE_VALUETYPE && !isNativePrimitiveStructType(pMTRetType))
+            {
+                rewriteReturnTypeToForceRetBuf = true;
+            }
+        }
+    }
+#endif // defined(TARGET_WINDOWS)
+
+    // Rewrite ExplicitThis to HasThis. This allows us to use ArgIterator which is unaware of ExplicitThis
+    // in the places where it is needed such as computation of return buffers.
+    if (sig.GetCallingConventionInfo() & IMAGE_CEE_CS_CALLCONV_EXPLICITTHIS)
+    {
+#if LOG_COMPUTE_CALL_STUB
+        printf("Managed ExplicitThis to HasThis conversion needed\n");
+#endif // LOG_COMPUTE_CALL_STUB
+        rewriteMetaSigFromExplicitThisToHasThis = true;
+    }
+
+    SigBuilder sigBuilder;
+    if (rewriteMetaSigFromExplicitThisToHasThis)
+    {
+#if LOG_COMPUTE_CALL_STUB
+        printf("Rewriting ExplicitThis to implicit this\n");
+#endif // LOG_COMPUTE_CALL_STUB
+        sigBuilder.AppendByte(IMAGE_CEE_CS_CALLCONV_DEFAULT_HASTHIS);
+        if ((sig.NumFixedArgs() == 0) || (sig.HasThis() && !sig.HasExplicitThis()))
+        {
+            ThrowHR(COR_E_BADIMAGEFORMAT);
+        }
+        sigBuilder.AppendData(sig.NumFixedArgs() - 1);
+        TypeHandle thRetType = sig.GetRetTypeHandleThrowing();
+#if defined(TARGET_WINDOWS)
+        if (rewriteReturnTypeToForceRetBuf)
+        {
+            // Change the return type to type large enough it will always need to be returned via return buffer
+            thRetType = CoreLibBinder::GetClass(CLASS__STACKFRAMEITERATOR);
+            _ASSERTE(thRetType.IsValueType());
+            _ASSERTE(thRetType.GetSize() > 64);
+            sigBuilder.AppendElementType(ELEMENT_TYPE_INTERNAL);
+            sigBuilder.AppendPointer(thRetType.AsPtr());
+        }
+        else
+#endif
+        {
+            SigPointer pReturn = sig.GetReturnProps();
+            pReturn.ConvertToInternalExactlyOne(sig.GetModule(), sig.GetSigTypeContext(), &sigBuilder);
+        }
+
+        // Skip the explicit this argument
+        sig.NextArg();
+
+        // Copy rest of the arguments
+        sig.NextArg();
+        SigPointer pArgs = sig.GetArgProps();
+        for (unsigned i = 1; i < sig.NumFixedArgs(); i++)
+        {
+            pArgs.ConvertToInternalExactlyOne(sig.GetModule(), sig.GetSigTypeContext(), &sigBuilder);
+        }
+
+        DWORD cSig;
+        PCCOR_SIGNATURE pNewSig = (PCCOR_SIGNATURE)sigBuilder.GetSignature(&cSig);
+        MetaSig newSig(pNewSig, cSig, sig.GetModule(), NULL, MetaSig::sigMember);
+        sig = newSig;
+    }
+
     ArgIterator argIt(&sig);
     int32_t interpreterStackOffset = 0;
 

src/coreclr/vm/callstubgenerator.h

@@ -179,7 +179,7 @@ class CallStubGenerator
         // The size of the routines array is three times the number of arguments plus one slot for the target method pointer.
         return sizeof(CallStubHeader) + ((numArgs + 1) * 3 + 1) * sizeof(PCODE);
     }
-    void ComputeCallStub(MetaSig &sig, PCODE *pRoutines);
+    void ComputeCallStub(MetaSig &sig, PCODE *pRoutines, MethodDesc *pMD);
 
     void TerminateCurrentRoutineIfNotOfNewType(RoutineType type, PCODE *pRoutines);
 };

src/coreclr/vm/siginfo.cpp

@@ -23,6 +23,8 @@
 #include <corhlprpriv.h>
 #include "argdestination.h"
 #include "multicorejit.h"
+#include "callconvbuilder.hpp"
+#include "dynamicmethod.h"
 
 /*******************************************************************/
 const CorTypeInfo::CorTypeInfoEntry CorTypeInfo::info[ELEMENT_TYPE_MAX] =
@@ -5903,3 +5905,26 @@ BOOL CompareTypeLayout(mdToken tk1, mdToken tk2, Module *pModule1, Module *pModu
 
     return TRUE;
 }
+
+#ifndef DACCESS_COMPILE
+CorInfoCallConvExtension GetUnmanagedCallConvExtension(MetaSig* pSig)
+{
+    STANDARD_VM_CONTRACT;
+    CallConvBuilder builder;
+    UINT errorResID;
+
+    HRESULT hr = CallConv::TryGetUnmanagedCallingConventionFromModOptSigStartingAtRetType(GetScopeHandle(pSig->GetModule()), pSig->GetReturnProps(), &builder, &errorResID);
+
+    if (FAILED(hr))
+        COMPlusThrowHR(hr, errorResID);
+
+    CorInfoCallConvExtension callConvLocal = builder.GetCurrentCallConv();
+
+    if (callConvLocal == CallConvBuilder::UnsetValue)
+    {
+        callConvLocal = CallConv::GetDefaultUnmanagedCallingConvention();
+    }
+
+    return callConvLocal;
+}
+#endif // DACCESS_COMPILE

src/coreclr/vm/siginfo.hpp

@@ -1208,6 +1208,8 @@ BOOL CompareTypeDefsForEquivalence(mdToken tk1, mdToken tk2, Module *pModule1, M
 BOOL IsTypeDefEquivalent(mdToken tk, Module *pModule);
 BOOL IsTypeDefExternallyVisible(mdToken tk, Module *pModule, DWORD dwAttrs);
 
+CorInfoCallConvExtension GetUnmanagedCallConvExtension(MetaSig* pSig);
+
 void ReportPointersFromValueType(promote_func *fn, ScanContext *sc, PTR_MethodTable pMT, PTR_VOID pSrc);
 
 #endif /* _H_SIGINFO */