Remove all explicit GC Poll calls in unmanaged code.

src/coreclr/System.Private.CoreLib/src/System/Buffer.CoreCLR.cs

@@ -10,13 +10,13 @@ namespace System
     public partial class Buffer
     {
         [LibraryImport(RuntimeHelpers.QCall, EntryPoint = "Buffer_Clear")]
-        private static unsafe partial void __ZeroMemory(void* b, nuint byteLength);
+        private static unsafe partial void ZeroMemoryInternal(void* b, nuint byteLength);
 
         [LibraryImport(RuntimeHelpers.QCall, EntryPoint = "Buffer_MemMove")]
-        private static unsafe partial void __Memmove(byte* dest, byte* src, nuint len);
+        private static unsafe partial void MemmoveInternal(byte* dest, byte* src, nuint len);
 
         [MethodImpl(MethodImplOptions.InternalCall)]
-        private static extern void __BulkMoveWithWriteBarrier(ref byte destination, ref byte source, nuint byteCount);
+        private static extern void BulkMoveWithWriteBarrierInternal(ref byte destination, ref byte source, nuint byteCount);
 
         // Used by ilmarshalers.cpp
         internal static unsafe void Memcpy(byte* dest, byte* src, int len)

src/coreclr/System.Private.CoreLib/src/System/GC.CoreCLR.cs

@@ -2,14 +2,9 @@
 // The .NET Foundation licenses this file to you under the MIT license.
 
 /*============================================================
-**
-**
-**
 ** Purpose: Exposes features of the Garbage Collector through
 ** the class libraries.  This is a class which cannot be
 ** instantiated.
-**
-**
 ===========================================================*/
 
 using System.Collections.Generic;
@@ -347,13 +342,17 @@ public static void WaitForPendingFinalizers()
         // Indicates that the system should not call the Finalize() method on
         // an object that would normally require this call.
         [MethodImpl(MethodImplOptions.InternalCall)]
-        private static extern void _SuppressFinalize(object o);
+        private static extern void SuppressFinalizeInternal(object o);
 
-        public static void SuppressFinalize(object obj)
+        public static unsafe void SuppressFinalize(object obj)
         {
             ArgumentNullException.ThrowIfNull(obj);
 
-            _SuppressFinalize(obj);
+            MethodTable* pMT = RuntimeHelpers.GetMethodTable(obj);
+            if (pMT->HasFinalizer)
+            {
+                SuppressFinalizeInternal(obj);
+            }
         }
 
         // Indicates that the system should call the Finalize() method on an object
@@ -600,12 +599,14 @@ private static bool InvokeMemoryLoadChangeNotifications()
                     return true;
                 }
 
-                // We need to take a snapshot of s_notifications.Count, so that in the case that s_notifications[i].Notification() registers new notifications,
-                // we neither get rid of them nor iterate over them
+                // We need to take a snapshot of s_notifications.Count, so that in the case that
+                // s_notifications[i].Notification() registers new notifications, we neither get rid
+                // of them nor iterate over them.
                 int count = s_notifications.Count;
 
-                // If there is no existing notifications, we won't be iterating over any and we won't be adding any new one. Also, there wasn't any added since
-                // we last invoked this method so it's safe to assume we can reset s_previousMemoryLoad.
+                // If there is no existing notifications, we won't be iterating over any and we won't
+                // be adding any new one. Also, there wasn't any added since we last invoked this
+                // method so it's safe to assume we can reset s_previousMemoryLoad.
                 if (count == 0)
                 {
                     s_previousMemoryLoad = float.MaxValue;

src/coreclr/System.Private.CoreLib/src/System/Threading/Thread.CoreCLR.cs

@@ -491,19 +491,24 @@ private void ResetFinalizerThreadSlow()
             }
         }
 
+        [MethodImpl(MethodImplOptions.InternalCall)]
+        private static extern bool CatchAtSafePoint();
+
         [LibraryImport(RuntimeHelpers.QCall, EntryPoint = "ThreadNative_PollGC")]
-        private static partial void ThreadNative_PollGC();
+        private static partial void PollGCInternal();
 
         // GC Suspension is done by simply dropping into native code via p/invoke, and we reuse the p/invoke
         // mechanism for suspension. On all architectures we should have the actual stub used for the check be implemented
         // as a small assembly stub which checks the global g_TrapReturningThreads flag and tail-call to this helper
         private static unsafe void PollGC()
         {
-            NativeThreadState catchAtSafePoint = ((NativeThreadClass*)Thread.DirectOnThreadLocalData.pNativeThread)->m_State & NativeThreadState.TS_CatchAtSafePoint;
-            if (catchAtSafePoint != NativeThreadState.None)
+            if (CatchAtSafePoint())
             {
-                ThreadNative_PollGC();
+                PollGCWorker();
             }
+
+            [MethodImpl(MethodImplOptions.NoInlining)]
+            static void PollGCWorker() => PollGCInternal();
         }
 
         [StructLayout(LayoutKind.Sequential)]

src/coreclr/classlibnative/bcltype/objectnative.cpp

@@ -83,8 +83,6 @@ FCIMPL2(FC_BOOL_RET, ObjectNative::ContentEquals, Object *pThisRef, Object *pCom
         pCompareRef->GetData(),
         pThisMT->GetNumInstanceFieldBytes()) == 0;
 
-    FC_GC_POLL_RET();
-
     FC_RETURN_BOOL(ret);
 }
 FCIMPLEND

src/coreclr/jit/compiler.h

@@ -11878,6 +11878,7 @@ class GenTreeVisitor
             case GT_IL_OFFSET:
             case GT_NOP:
             case GT_SWIFT_ERROR:
+            case GT_GCPOLL:
                 break;
 
             // Lclvar unary operators

src/coreclr/jit/compiler.hpp

@@ -4408,6 +4408,7 @@ void GenTree::VisitOperands(TVisitor visitor)
         case GT_IL_OFFSET:
         case GT_NOP:
         case GT_SWIFT_ERROR:
+        case GT_GCPOLL:
             return;
 
         // Unary operators with an optional operand

src/coreclr/jit/flowgraph.cpp

@@ -48,6 +48,10 @@ static bool blockNeedsGCPoll(BasicBlock* block)
                         blockMayNeedGCPoll = true;
                     }
                 }
+                else if (tree->OperGet() == GT_GCPOLL)
+                {
+                    blockMayNeedGCPoll = true;
+                }
             }
         }
     }

src/coreclr/jit/gentree.cpp

@@ -2670,6 +2670,7 @@ bool GenTree::Compare(GenTree* op1, GenTree* op2, bool swapOK)
             case GT_NOP:
             case GT_LABEL:
             case GT_SWIFT_ERROR:
+            case GT_GCPOLL:
                 return true;
 
             default:
@@ -5239,6 +5240,7 @@ unsigned Compiler::gtSetEvalOrder(GenTree* tree)
                 break;
 
             case GT_NOP:
+            case GT_GCPOLL:
                 level  = 0;
                 costEx = 0;
                 costSz = 0;
@@ -6584,6 +6586,7 @@ bool GenTree::TryGetUse(GenTree* operand, GenTree*** pUse)
         case GT_IL_OFFSET:
         case GT_NOP:
         case GT_SWIFT_ERROR:
+        case GT_GCPOLL:
             return false;
 
         // Standard unary operators
@@ -6912,6 +6915,7 @@ bool GenTree::OperRequiresCallFlag(Compiler* comp) const
         case GT_CALL:
             return true;
 
+        case GT_GCPOLL:
         case GT_KEEPALIVE:
             return true;
 
@@ -7243,6 +7247,7 @@ bool GenTree::OperRequiresGlobRefFlag(Compiler* comp) const
         case GT_MEMORYBARRIER:
         case GT_KEEPALIVE:
         case GT_SWIFT_ERROR:
+        case GT_GCPOLL:
             return true;
 
         case GT_CALL:
@@ -9430,6 +9435,7 @@ GenTree* Compiler::gtCloneExpr(GenTree* tree)
             case GT_NOP:
             case GT_LABEL:
             case GT_SWIFT_ERROR:
+            case GT_GCPOLL:
                 copy = new (this, oper) GenTree(oper, tree->gtType);
                 goto DONE;
 
@@ -10201,6 +10207,7 @@ GenTreeUseEdgeIterator::GenTreeUseEdgeIterator(GenTree* node)
         case GT_IL_OFFSET:
         case GT_NOP:
         case GT_SWIFT_ERROR:
+        case GT_GCPOLL:
             m_state = -1;
             return;
 
@@ -12349,6 +12356,7 @@ void Compiler::gtDispLeaf(GenTree* tree, IndentStack* indentStack)
         case GT_PINVOKE_PROLOG:
         case GT_JMPTABLE:
         case GT_SWIFT_ERROR:
+        case GT_GCPOLL:
             break;
 
         case GT_RET_EXPR:

src/coreclr/jit/gtlist.h

@@ -37,6 +37,7 @@ GTNODE(LABEL            , GenTree            ,0,0,GTK_LEAF)             // Jump-
 GTNODE(JMP              , GenTreeVal         ,0,0,GTK_LEAF|GTK_NOVALUE) // Jump to another function
 GTNODE(FTN_ADDR         , GenTreeFptrVal     ,0,0,GTK_LEAF)             // Address of a function
 GTNODE(RET_EXPR         , GenTreeRetExpr     ,0,0,GTK_LEAF|DBK_NOTLIR)  // Place holder for the return expression from an inline candidate
+GTNODE(GCPOLL           , GenTree            ,0,0,GTK_LEAF|GTK_NOVALUE|DBK_NOTLIR)
 
 //-----------------------------------------------------------------------------
 //  Constant nodes:

src/coreclr/jit/importercalls.cpp

@@ -4090,6 +4090,22 @@ GenTree* Compiler::impIntrinsic(CORINFO_CLASS_HANDLE    clsHnd,
                 break;
             }
 
+            case NI_System_Threading_Thread_FastPollGC:
+            {
+                optMethodFlags |= OMF_NEEDS_GCPOLLS;
+                compCurBB->SetFlags(BBF_NEEDS_GCPOLL);
+
+                GenTree* gcpoll = new (this, GT_GCPOLL) GenTree(GT_GCPOLL, TYP_VOID);
+                // Prevent both reordering and removal. Invalid optimizations of Thread.FastPollGC are
+                // very subtle and hard to observe. Thus we are conservatively marking it with both
+                // GTF_CALL and GTF_GLOB_REF side-effects even though it may be more than strictly
+                // necessary. The conservative side-effects are unlikely to have negative impact
+                // on code quality in this case.
+                gcpoll->gtFlags |= (GTF_CALL | GTF_GLOB_REF);
+                retNode = gcpoll;
+                break;
+            }
+
 #if defined(TARGET_ARM64) || defined(TARGET_RISCV64) || defined(TARGET_XARCH)
             case NI_System_Threading_Interlocked_Or:
             case NI_System_Threading_Interlocked_And:
@@ -11145,6 +11161,10 @@ NamedIntrinsic Compiler::lookupNamedIntrinsic(CORINFO_METHOD_HANDLE method)
                         {
                             result = NI_System_Threading_Thread_get_ManagedThreadId;
                         }
+                        else if (strcmp(methodName, "FastPollGC") == 0)
+                        {
+                            result = NI_System_Threading_Thread_FastPollGC;
+                        }
                     }
                     else if (strcmp(className, "Volatile") == 0)
                     {

src/coreclr/jit/liveness.cpp

@@ -1470,6 +1470,7 @@ void Compiler::fgComputeLifeLIR(VARSET_TP& life, BasicBlock* block, VARSET_VALAR
             case GT_IL_OFFSET:
             case GT_KEEPALIVE:
             case GT_SWIFT_ERROR_RET:
+            case GT_GCPOLL:
                 // Never remove these nodes, as they are always side-effecting.
                 //
                 // NOTE: the only side-effect of some of these nodes (GT_CMP, GT_SUB_HI) is a write to the flags

src/coreclr/jit/namedintrinsiclist.h

@@ -75,6 +75,7 @@ enum NamedIntrinsic : unsigned short
 
     NI_System_Threading_Thread_get_CurrentThread,
     NI_System_Threading_Thread_get_ManagedThreadId,
+    NI_System_Threading_Thread_FastPollGC,
     NI_System_Threading_Volatile_Read,
     NI_System_Threading_Volatile_Write,
     NI_System_Threading_Volatile_ReadBarrier,

src/coreclr/jit/optcse.cpp

@@ -1908,6 +1908,7 @@ bool CSE_HeuristicCommon::CanConsiderTree(GenTree* tree, bool isReturn)
         case GT_COLON:
         case GT_QMARK:
         case GT_NOP:
+        case GT_GCPOLL:
         case GT_RETURN:
             return false; // Currently the only special nodes that we hit
                           // that we know that we don't want to CSE

src/coreclr/jit/rationalize.cpp

@@ -738,6 +738,13 @@ Compiler::fgWalkResult Rationalizer::RewriteNode(GenTree** useEdge, Compiler::Ge
             }
             break;
 
+        case GT_GCPOLL:
+        {
+            // GCPOLL is essentially a no-op, we used it as a hint for fgCreateGCPoll
+            node->gtBashToNOP();
+            return Compiler::WALK_CONTINUE;
+        }
+
         case GT_COMMA:
         {
             GenTree*           op1         = node->gtGetOp1();

src/coreclr/jit/valuenum.cpp

@@ -12276,6 +12276,7 @@ void Compiler::fgValueNumberTree(GenTree* tree)
             // These do not represent values.
             case GT_NO_OP:
             case GT_NOP:
+            case GT_GCPOLL:
             case GT_JMP:   // Control flow
             case GT_LABEL: // Control flow
 #if defined(FEATURE_EH_WINDOWS_X86)

src/coreclr/nativeaot/System.Private.CoreLib/src/System/Buffer.NativeAot.cs

@@ -12,15 +12,15 @@ namespace System
     public static partial class Buffer
     {
         [MethodImpl(MethodImplOptions.AggressiveInlining)]
-        private static unsafe void __ZeroMemory(void* b, nuint byteLength) =>
+        private static unsafe void ZeroMemoryInternal(void* b, nuint byteLength) =>
             RuntimeImports.memset((byte*)b, 0, byteLength);
 
         [MethodImpl(MethodImplOptions.AggressiveInlining)]
-        private static unsafe void __Memmove(byte* dest, byte* src, nuint len) =>
+        private static unsafe void MemmoveInternal(byte* dest, byte* src, nuint len) =>
             RuntimeImports.memmove(dest, src, len);
 
         [MethodImpl(MethodImplOptions.AggressiveInlining)]
-        private static void __BulkMoveWithWriteBarrier(ref byte destination, ref byte source, nuint byteCount) =>
+        private static void BulkMoveWithWriteBarrierInternal(ref byte destination, ref byte source, nuint byteCount) =>
             RuntimeImports.RhBulkMoveWithWriteBarrier(ref destination, ref source, byteCount);
     }
 }

src/coreclr/vm/comsynchronizable.cpp

@@ -714,6 +714,14 @@ FCIMPL1(void, ThreadNative::Finalize, ThreadBaseObject* pThisUNSAFE)
 }
 FCIMPLEND
 
+FCIMPL0(FC_BOOL_RET, ThreadNative::CatchAtSafePoint)
+{
+    FCALL_CONTRACT;
+
+    FC_RETURN_BOOL(GetThread()->CatchAtSafePoint());
+}
+FCIMPLEND
+
 // Get whether or not this is a background thread.
 extern "C" BOOL QCALLTYPE ThreadNative_GetIsBackground(QCall::ThreadHandle thread)
 {

src/coreclr/vm/comsynchronizable.h

@@ -39,8 +39,9 @@ class ThreadNative
         ThreadAbortRequested = 128,
     };
 
-    static FCDECL0(INT32,   GetOptimalMaxSpinWaitsPerSpinIteration);
-    static FCDECL1(void,    Finalize,                       ThreadBaseObject* pThis);
+    static FCDECL0(INT32,       GetOptimalMaxSpinWaitsPerSpinIteration);
+    static FCDECL1(void,        Finalize, ThreadBaseObject* pThis);
+    static FCDECL0(FC_BOOL_RET, CatchAtSafePoint);
 };
 
 extern "C" void QCALLTYPE ThreadNative_Start(QCall::ThreadHandle thread, int threadStackSize, int priority, BOOL isThreadPool, PCWSTR pThreadName);