GenericPInvokeCalli shift left then or 1 (dotnet#17734)

When _WIN64 is defined vm relies on the secret arg being
shifted left and orred with #1.

Revert part of changes from dotnet#17659 to fix dotnet/corefx#29266

Fix arm64 to match amd64

Simplify dllimport.cpp

Author: sdmaclea

src/vm/amd64/PInvokeStubs.asm

@@ -31,6 +31,14 @@ LEAF_ENTRY GenericPInvokeCalliHelper, _TEXT
         test            rax, rax
         jz              GenericPInvokeCalliGenILStub
 
+        ;
+        ; We need to distinguish between a MethodDesc* and an unmanaged target.
+        ; The way we do this is to shift the managed target to the left by one bit and then set the
+        ; least significant bit to 1.  This works because MethodDesc* are always 8-byte aligned.
+        ;
+        shl             PINVOKE_CALLI_TARGET_REGISTER, 1
+        or              PINVOKE_CALLI_TARGET_REGISTER, 1
+
         ;
         ; jump to existing IL stub
         ;

src/vm/amd64/pinvokestubs.S

@@ -24,6 +24,14 @@ LEAF_ENTRY GenericPInvokeCalliHelper, _TEXT
         test            rax, rax
         jz              C_FUNC(GenericPInvokeCalliGenILStub)
 
+        //
+        // We need to distinguish between a MethodDesc* and an unmanaged target.
+        // The way we do this is to shift the managed target to the left by one bit and then set the
+        // least significant bit to 1.  This works because MethodDesc* are always 8-byte aligned.
+        //
+        shl             PINVOKE_CALLI_TARGET_REGISTER, 1
+        or              PINVOKE_CALLI_TARGET_REGISTER, 1
+
         //
         // jump to existing IL stub
         //

src/vm/arm64/PInvokeStubs.asm

@@ -58,6 +58,15 @@ __PInvokeGenStubFuncName SETS "$__PInvokeGenStubFuncName":CC:"_RetBuffArg"
         ; if null goto stub generation
         cbz                 x9, %0
 
+        IF "$FuncPrefix" == "GenericPInvokeCalli"
+            ;
+            ; We need to distinguish between a MethodDesc* and an unmanaged target.
+            ; The way we do this is to shift the managed target to the left by one bit and then set the
+            ; least significant bit to 1.  This works because MethodDesc* are always 8-byte aligned.
+            ;
+            lsl             x9, x9, #1
+            orr             x9, x9, #1
+        ENDIF
 
         EPILOG_BRANCH_REG   x9 
 

src/vm/arm64/pinvokestubs.S

@@ -21,7 +21,7 @@
 // $VASigCookieReg : register which contains the VASigCookie
 // $SaveFPArgs : "Yes" or "No" . For varidic functions FP Args are not present in FP regs 
 //                        So need not save FP Args registers for vararg Pinvoke
-.macro PINVOKE_STUB __PInvokeStubFuncName,__PInvokeGenStubFuncName,__PInvokeStubWorkerName,VASigCookieReg,HiddenArg,SaveFPArgs
+.macro PINVOKE_STUB __PInvokeStubFuncName,__PInvokeGenStubFuncName,__PInvokeStubWorkerName,VASigCookieReg,HiddenArg,SaveFPArgs,ShiftLeftAndOrSecret=0
 
 
         NESTED_ENTRY \__PInvokeStubFuncName, _TEXT, NoHandler
@@ -32,6 +32,16 @@
         // if null goto stub generation
         cbz                 x9, LOCAL_LABEL(\__PInvokeStubFuncName\()_0)
 
+        .if (\ShiftLeftAndOrSecret == 1)
+            //
+            // We need to distinguish between a MethodDesc* and an unmanaged target.
+            // The way we do this is to shift the managed target to the left by one bit and then set the
+            // least significant bit to 1.  This works because MethodDesc* are always 8-byte aligned.
+            //
+            lsl             x9, x9, #1
+            orr             x9, x9, #1
+        .endif
+
         EPILOG_BRANCH_REG   x9 
 
 LOCAL_LABEL(\__PInvokeStubFuncName\()_0):
@@ -97,7 +107,7 @@ PINVOKE_STUB VarargPInvokeStub, VarargPInvokeGenILStub, VarargPInvokeStubWorker,
 // x15 = VASigCookie*
 // x12 = Unmanaged target
 //
-PINVOKE_STUB GenericPInvokeCalliHelper, GenericPInvokeCalliGenILStub, GenericPInvokeCalliStubWorker, x15, x12, 1
+PINVOKE_STUB GenericPInvokeCalliHelper, GenericPInvokeCalliGenILStub, GenericPInvokeCalliStubWorker, x15, x12, 1, 1
 
 // ------------------------------------------------------------------
 // VarargPInvokeStub_RetBuffArg & VarargPInvokeGenILStub_RetBuffArg

src/vm/dllimport.cpp

@@ -2256,6 +2256,11 @@ void NDirectStubLinker::DoNDirect(ILCodeStream *pcsEmit, DWORD dwStubFlags, Meth
                 // for managed-to-unmanaged CALLI that requires marshaling, the target is passed
                 // as the secret argument to the stub by GenericPInvokeCalliHelper (asmhelpers.asm)
                 EmitLoadStubContext(pcsEmit, dwStubFlags);
+#ifdef _WIN64
+                // the secret arg has been shifted to left and ORed with 1 (see code:GenericPInvokeCalliHelper)
+                pcsEmit->EmitLDC(1);
+                pcsEmit->EmitSHR_UN();
+#endif
             }
             else
 #ifdef FEATURE_COMINTEROP

src/vm/frames.h

@@ -2957,6 +2957,7 @@ class InlinedCallFrame : public Frame
         WRAPPER_NO_CONTRACT;
 
 #ifdef _WIN64
+        // See code:GenericPInvokeCalliHelper
         return ((m_Datum != NULL) && !(dac_cast<TADDR>(m_Datum) & 0x1));
 #else // _WIN64
         return ((dac_cast<TADDR>(m_Datum) & ~0xffff) != 0);

src/vm/stubhelpers.cpp

@@ -1166,6 +1166,7 @@ FCIMPL2(void*, StubHelpers::GetDelegateTarget, DelegateObject *pThisUNSAFE, UINT
 #if defined(_WIN64)
     UINT_PTR target = (UINT_PTR)orefThis->GetMethodPtrAux();
 
+    // See code:GenericPInvokeCalliHelper
     // The lowest bit is used to distinguish between MD and target on 64-bit.
     target = (target << 1) | 1;