Fix x64 and x86 emulation on arm64 Windows (dotnet#72693)

* Fix x64 and x86 emulation on arm64 Windows

The runtime was hanging or crashing when running on x64 and x86
emulation on Windows ARM64. Most of these failures were caused by
missing cache flushes after code updates where the emulator kept
executing arm64 code that it has jitted from the previous state of
the code.

There were also two CONTEXT related issues. One was that we were
assuming that AVX state is always supported. However, the emulator
doesn't support it, so SetXStateFeaturesMask(pCtx, XSTATE_MASK_AVX)
was failing and we have considered that a failure to capture thread
context.
The other one was that we have assumed that the CONTEXT we get in
Thread::RedirectCurrentThreadAtHandledJITCase is CONTEXT_COMPLETE,
but with the emulation, we one get CONTEXT_FULL, that means
CONTEXT_COMPLETE without the debug registers.

There were also two CoreCLR tests issues that caused failures under the
emulation, I have fixed those.

* Add hasCodeExecutedBefore argument to ClrFlushInstructionCache

Author: janvorli

src/coreclr/vm/amd64/cgenamd64.cpp

@@ -548,6 +548,7 @@ void UMEntryThunkCode::Encode(UMEntryThunkCode *pEntryThunkCodeRX, BYTE* pTarget
     m_jmpRAX[2]  = 0xE0;
 
     _ASSERTE(DbgIsExecutable(&pEntryThunkCodeRX->m_movR10[0], &pEntryThunkCodeRX->m_jmpRAX[3]-&pEntryThunkCodeRX->m_movR10[0]));
+    FlushInstructionCache(GetCurrentProcess(),pEntryThunkCodeRX,sizeof(UMEntryThunkCode));
 }
 
 void UMEntryThunkCode::Poison()
@@ -574,7 +575,7 @@ void UMEntryThunkCode::Poison()
     pThisRW->m_movR10[1]  = 0xBF;
 #endif
 
-    ClrFlushInstructionCache(&m_movR10[0], &m_jmpRAX[3]-&m_movR10[0]);
+    ClrFlushInstructionCache(&m_movR10[0], &m_jmpRAX[3]-&m_movR10[0], /* hasCodeExecutedBefore */ true);
 }
 
 UMEntryThunk* UMEntryThunk::Decode(LPVOID pCallback)

src/coreclr/vm/amd64/cgencpu.h

@@ -534,10 +534,16 @@ DWORD GetOffsetAtEndOfFunction(ULONGLONG           uImageBase,
 // Currently ClrFlushInstructionCache has no effect on AMD64
 //
 
-inline BOOL ClrFlushInstructionCache(LPCVOID pCodeAddr, size_t sizeOfCode)
+inline BOOL ClrFlushInstructionCache(LPCVOID pCodeAddr, size_t sizeOfCode, bool hasCodeExecutedBefore = false)
 {
-    // FlushInstructionCache(GetCurrentProcess(), pCodeAddr, sizeOfCode);
-    MemoryBarrier();
+    if (hasCodeExecutedBefore)
+    {
+        FlushInstructionCache(GetCurrentProcess(), pCodeAddr, sizeOfCode);
+    }
+    else
+    {
+        MemoryBarrier();
+    }
     return TRUE;
 }
 

src/coreclr/vm/amd64/jitinterfaceamd64.cpp

@@ -296,6 +296,7 @@ int WriteBarrierManager::ChangeWriteBarrierTo(WriteBarrierType newWriteBarrier,
     {
         ExecutableWriterHolder<void> writeBarrierWriterHolder(GetWriteBarrierCodeLocation((void*)JIT_WriteBarrier), GetCurrentWriteBarrierSize());
         memcpy(writeBarrierWriterHolder.GetRW(), (LPVOID)GetCurrentWriteBarrierCode(), GetCurrentWriteBarrierSize());
+        stompWBCompleteActions |= SWB_ICACHE_FLUSH;
     }
 
     switch (newWriteBarrier)

src/coreclr/vm/arm/cgencpu.h

@@ -998,7 +998,7 @@ struct HijackArgs
 // Currently ClrFlushInstructionCache has no effect on X86
 //
 
-inline BOOL ClrFlushInstructionCache(LPCVOID pCodeAddr, size_t sizeOfCode)
+inline BOOL ClrFlushInstructionCache(LPCVOID pCodeAddr, size_t sizeOfCode, bool hasCodeExecutedBefore = false)
 {
     return FlushInstructionCache(GetCurrentProcess(), pCodeAddr, sizeOfCode);
 }

src/coreclr/vm/arm64/cgencpu.h

@@ -277,7 +277,7 @@ inline NEON128 GetSimdMem(PCODE ip)
 void emitCOMStubCall (ComCallMethodDesc *pCOMMethodRX, ComCallMethodDesc *pCOMMethodRW, PCODE target);
 #endif // FEATURE_COMINTEROP
 
-inline BOOL ClrFlushInstructionCache(LPCVOID pCodeAddr, size_t sizeOfCode)
+inline BOOL ClrFlushInstructionCache(LPCVOID pCodeAddr, size_t sizeOfCode, bool hasCodeExecutedBefore = false)
 {
     return FlushInstructionCache(GetCurrentProcess(), pCodeAddr, sizeOfCode);
 }

src/coreclr/vm/comcallablewrapper.cpp

@@ -566,6 +566,7 @@ extern "C" PCODE ComPreStubWorker(ComPrestubMethodFrame *pPFrame, UINT64 *pError
 #else
             *ppofsWriterHolder.GetRW() = ((UINT_PTR)pStub);
 #endif
+            ClrFlushInstructionCache(ppofs, sizeof(UINT_PTR), /* hasCodeExecutedBefore */ true);
 
             // Return the address of the prepad. The prepad will regenerate the hidden parameter and due
             // to the update above will execute the new stub code the second time around.

src/coreclr/vm/i386/cgencpu.h

@@ -501,10 +501,16 @@ struct HijackArgs
 // Currently ClrFlushInstructionCache has no effect on X86
 //
 
-inline BOOL ClrFlushInstructionCache(LPCVOID pCodeAddr, size_t sizeOfCode)
+inline BOOL ClrFlushInstructionCache(LPCVOID pCodeAddr, size_t sizeOfCode, bool hasCodeExecutedBefore = false)
 {
-    // FlushInstructionCache(GetCurrentProcess(), pCodeAddr, sizeOfCode);
-    MemoryBarrier();
+    if (hasCodeExecutedBefore)
+    {
+        FlushInstructionCache(GetCurrentProcess(), pCodeAddr, sizeOfCode);
+    }
+    else
+    {
+        MemoryBarrier();
+    }
     return TRUE;
 }
 

src/coreclr/vm/i386/cgenx86.cpp

@@ -1157,7 +1157,7 @@ void UMEntryThunkCode::Encode(UMEntryThunkCode *pEntryThunkCodeRX, BYTE* pTarget
     m_jmp        = X86_INSTR_JMP_REL32;
     m_execstub   = (BYTE*) ((pTargetCode) - (4+((BYTE*)&pEntryThunkCodeRX->m_execstub)));
 
-    FlushInstructionCache(GetCurrentProcess(),pEntryThunkCodeRX->GetEntryPoint(),sizeof(UMEntryThunkCode));
+    ClrFlushInstructionCache(pEntryThunkCodeRX->GetEntryPoint(),sizeof(UMEntryThunkCode) - GetEntryPointOffset(), /* hasCodeExecutedBefore */ true);
 }
 
 void UMEntryThunkCode::Poison()
@@ -1172,7 +1172,7 @@ void UMEntryThunkCode::Poison()
     // mov ecx, imm32
     pThisRW->m_movEAX = 0xb9;
 
-    ClrFlushInstructionCache(GetEntryPoint(),sizeof(UMEntryThunkCode));
+    ClrFlushInstructionCache(GetEntryPoint(),sizeof(UMEntryThunkCode) - GetEntryPointOffset(), /* hasCodeExecutedBefore */ true);
 }
 
 UMEntryThunk* UMEntryThunk::Decode(LPVOID pCallback)

src/coreclr/vm/i386/jitinterfacex86.cpp

@@ -1326,7 +1326,11 @@ int StompWriteBarrierResize(bool isRuntimeSuspended, bool bReqUpperBoundsCheck)
                 // cmp offset[edx], 0ffh instruction
                 _ASSERTE(pBuf[22] == 0x80);
                 pfunc = (size_t *) &pBufRW[PostGrow_CardTableFirstLocation];
-               *pfunc = (size_t) g_card_table;
+                if (*pfunc != (size_t) g_card_table)
+                {
+                    stompWBCompleteActions |= SWB_ICACHE_FLUSH;
+                    *pfunc = (size_t) g_card_table;
+                }
 
                 // What we're trying to update is the offset field of a
                 // mov offset[edx], 0ffh instruction
@@ -1341,7 +1345,11 @@ int StompWriteBarrierResize(bool isRuntimeSuspended, bool bReqUpperBoundsCheck)
                 // cmp offset[edx], 0ffh instruction
                 _ASSERTE(pBuf[14] == 0x80);
                 pfunc = (size_t *) &pBufRW[PreGrow_CardTableFirstLocation];
-               *pfunc = (size_t) g_card_table;
+                if (*pfunc != (size_t) g_card_table)
+                {
+                    stompWBCompleteActions |= SWB_ICACHE_FLUSH;
+                    *pfunc = (size_t) g_card_table;
+                }
 
                 // What we're trying to update is the offset field of a
 
@@ -1357,7 +1365,11 @@ int StompWriteBarrierResize(bool isRuntimeSuspended, bool bReqUpperBoundsCheck)
             // cmp offset[edx], 0ffh instruction
             _ASSERTE(pBuf[22] == 0x80);
             pfunc = (size_t *) &pBufRW[PostGrow_CardTableFirstLocation];
-           *pfunc = (size_t) g_card_table;
+            if (*pfunc != (size_t) g_card_table)
+            {
+                stompWBCompleteActions |= SWB_ICACHE_FLUSH;
+                *pfunc = (size_t) g_card_table;
+            }
 
             // What we're trying to update is the offset field of a
             // mov offset[edx], 0ffh instruction
@@ -1366,7 +1378,11 @@ int StompWriteBarrierResize(bool isRuntimeSuspended, bool bReqUpperBoundsCheck)
         }
 
         // Stick in the adjustment value.
-        *pfunc = (size_t) g_card_table;
+        if (*pfunc != (size_t) g_card_table)
+        {
+            stompWBCompleteActions |= SWB_ICACHE_FLUSH;
+            *pfunc = (size_t) g_card_table;
+        }
     }
 
     if (bStompWriteBarrierEphemeral)
@@ -1379,7 +1395,7 @@ int StompWriteBarrierResize(bool isRuntimeSuspended, bool bReqUpperBoundsCheck)
 
 void FlushWriteBarrierInstructionCache()
 {
-    FlushInstructionCache(GetCurrentProcess(), (void *)JIT_PatchedWriteBarrierGroup,
-        (BYTE*)JIT_PatchedWriteBarrierGroup_End - (BYTE*)JIT_PatchedWriteBarrierGroup);
+    ClrFlushInstructionCache(GetWriteBarrierCodeLocation((BYTE*)JIT_PatchedWriteBarrierGroup),
+        (BYTE*)JIT_PatchedWriteBarrierGroup_End - (BYTE*)JIT_PatchedWriteBarrierGroup, /* hasCodeExecutedBefore */ true);
 }
 

src/coreclr/vm/jitinterface.cpp

@@ -12954,7 +12954,8 @@ PCODE UnsafeJitFunction(PrepareCodeConfig* config,
         //DbgPrintf("Jitted Entry at" FMT_ADDR "method %s::%s %s size %08x\n", DBG_ADDR(nativeEntry),
         //          pszDebugClassName, pszDebugMethodName, pszDebugMethodSignature, sizeOfCode);
 
-        ClrFlushInstructionCache(nativeEntry, sizeOfCode);
+        // For dynamic method, the code memory may be reused, thus we are passing in the hasCodeExecutedBefore set to true
+        ClrFlushInstructionCache(nativeEntry, sizeOfCode, /* hasCodeExecutedBefore */ true);
         ret = (PCODE)nativeEntry;
 
 #ifdef TARGET_ARM

src/coreclr/vm/loongarch64/cgencpu.h

@@ -226,7 +226,7 @@ inline TADDR GetMem(PCODE address, SIZE_T size, bool signExtend)
 void emitCOMStubCall (ComCallMethodDesc *pCOMMethodRX, ComCallMethodDesc *pCOMMethodRW, PCODE target);
 #endif // FEATURE_COMINTEROP
 
-inline BOOL ClrFlushInstructionCache(LPCVOID pCodeAddr, size_t sizeOfCode)
+inline BOOL ClrFlushInstructionCache(LPCVOID pCodeAddr, size_t sizeOfCode, bool hasCodeExecutedBefore = false)
 {
     return FlushInstructionCache(GetCurrentProcess(), pCodeAddr, sizeOfCode);
 }

src/coreclr/vm/precode.cpp

@@ -350,6 +350,7 @@ BOOL Precode::SetTargetInterlocked(PCODE target, BOOL fOnlyRedirectFromPrestub)
 #ifdef HAS_THISPTR_RETBUF_PRECODE
     case PRECODE_THISPTR_RETBUF:
         ret = AsThisPtrRetBufPrecode()->SetTargetInterlocked(target, expected);
+        ClrFlushInstructionCache(this, sizeof(ThisPtrRetBufPrecode), /* hasCodeExecutedBefore */ true);
         break;
 #endif // HAS_THISPTR_RETBUF_PRECODE
 
@@ -381,7 +382,7 @@ void Precode::Reset()
     {
         ExecutableWriterHolder<Precode> precodeWriterHolder(this, size);
         precodeWriterHolder.GetRW()->Init(this, t, pMD, pMD->GetLoaderAllocator());
-        ClrFlushInstructionCache(this, SizeOf());
+        ClrFlushInstructionCache(this, SizeOf(), /* hasCodeExecutedBefore */ true);
     }
 }
 

src/coreclr/vm/threadsuspend.cpp

@@ -2880,7 +2880,7 @@ BOOL Thread::RedirectThreadAtHandledJITCase(PFN_REDIRECTTARGET pTgt)
     // This should not normally fail.
     // The system silently ignores any feature specified in the FeatureMask
     // which is not enabled on the processor.
-    bRes &= SetXStateFeaturesMask(pCtx, XSTATE_MASK_AVX);
+    SetXStateFeaturesMask(pCtx, XSTATE_MASK_AVX);
 #endif //defined(TARGET_X86) || defined(TARGET_AMD64)
 
     // Make sure we specify CONTEXT_EXCEPTION_REQUEST to detect "trap frame reporting".
@@ -2981,7 +2981,7 @@ BOOL Thread::RedirectCurrentThreadAtHandledJITCase(PFN_REDIRECTTARGET pTgt, CONT
     _ASSERTE(PreemptiveGCDisabledOther());
     _ASSERTE(IsAddrOfRedirectFunc(pTgt));
     _ASSERTE(pCurrentThreadCtx);
-    _ASSERTE((pCurrentThreadCtx->ContextFlags & CONTEXT_COMPLETE) == CONTEXT_COMPLETE);
+    _ASSERTE((pCurrentThreadCtx->ContextFlags & CONTEXT_FULL) == CONTEXT_FULL);
     _ASSERTE(ExecutionManager::IsManagedCode(GetIP(pCurrentThreadCtx)));
 
     ////////////////////////////////////////////////////////////////

src/tests/JIT/HardwareIntrinsics/X86/X86Base/CpuId.cs

@@ -28,11 +28,12 @@ static unsafe int Main(string[] args)
 
             bool isAuthenticAmd = (ebx == 0x68747541) && (ecx == 0x444D4163) && (edx == 0x69746E65);
             bool isGenuineIntel = (ebx == 0x756E6547) && (ecx == 0x6C65746E) && (edx == 0x49656E69);
+            bool isVirtualCPU = (ebx == 0x74726956) && (ecx == 0x20555043) && (edx == 0x206C6175);
 
-            if (!isAuthenticAmd && !isGenuineIntel)
+            if (!isAuthenticAmd && !isGenuineIntel && !isVirtualCPU)
             {
                 // CPUID checks are vendor specific and aren't guaranteed to match up, even across Intel/AMD
-                // as such, we limit ourselves to just AuthenticAMD and GenuineIntel right now. Any other
+                // as such, we limit ourselves to just AuthenticAMD, GenuineIntel and "Virtual CPU" right now. Any other
                 // vendors would need to be validated against the checks below and added to the list as necessary.
 
                 // An example of a difference is Intel/AMD for LZCNT. While the same underlying bit is used to

src/tests/JIT/Regression/JitBlue/Runtime_34587/Runtime_34587.cs

@@ -422,13 +422,13 @@ static bool ValidateX86Base()
 
             if (X86BaseIsSupported)
             {
-                succeeded &= (RuntimeInformation.OSArchitecture == Architecture.X86) || (RuntimeInformation.OSArchitecture == Architecture.X64);
+                succeeded &= (RuntimeInformation.ProcessArchitecture == Architecture.X86) || (RuntimeInformation.ProcessArchitecture == Architecture.X64);
             }
 
             if (X86BaseX64IsSupported)
             {
                 succeeded &= X86BaseIsSupported;
-                succeeded &= (RuntimeInformation.OSArchitecture == Architecture.X64);
+                succeeded &= (RuntimeInformation.ProcessArchitecture == Architecture.X64);
             }
 
             return succeeded;