Add support for Windows IO completions to the portable thread pool (#64834)

* Add support for Windows IO completions to the portable thread pool

- Added an implementation for BindHandle
- Polling for IO completions is done in batches on separate threads similarly to what is done on Unixes
- Added a high-priority work item queue to have IO completion work items run at higher priority
- Removed the time-sensitive work item queue, used the high-priority queue instead

Author: kouvel

src/coreclr/System.Private.CoreLib/System.Private.CoreLib.csproj

@@ -232,9 +232,6 @@
     <Compile Include="$(BclSourcesRoot)\System\String.CoreCLR.cs" />
     <Compile Include="$(BclSourcesRoot)\System\StubHelpers.cs" />
     <Compile Include="$(BclSourcesRoot)\System\Text\StringBuilder.CoreCLR.cs" />
-    <Compile Include="$(BclSourcesRoot)\System\Threading\ClrThreadPoolBoundHandle.cs" />
-    <Compile Include="$(BclSourcesRoot)\System\Threading\ClrThreadPoolBoundHandleOverlapped.cs" />
-    <Compile Include="$(BclSourcesRoot)\System\Threading\ClrThreadPoolPreAllocatedOverlapped.cs" />
     <Compile Include="$(BclSourcesRoot)\System\Threading\Interlocked.CoreCLR.cs" />
     <Compile Include="$(BclSourcesRoot)\System\Threading\Monitor.CoreCLR.cs" />
     <Compile Include="$(BclSourcesRoot)\System\Threading\Overlapped.cs" />
@@ -289,14 +286,13 @@
   </ItemGroup>
   <ItemGroup Condition="'$(TargetsUnix)' == 'true'">
     <Compile Include="$(BclSourcesRoot)\Interop\Unix\Interop.Libraries.cs" />
-    <Compile Include="$(BclSourcesRoot)\System\Threading\ClrThreadPoolBoundHandle.Unix.cs" />
     <Compile Include="$(BclSourcesRoot)\System\Threading\LowLevelLifoSemaphore.Unix.cs" />
   </ItemGroup>
   <ItemGroup Condition="'$(TargetsWindows)' == 'true'">
     <Compile Include="$(CommonPath)Interop\Windows\OleAut32\Interop.VariantClear.cs">
       <Link>Common\Interop\Windows\OleAut32\Interop.VariantClear.cs</Link>
     </Compile>
-    <Compile Include="$(BclSourcesRoot)\System\Threading\ClrThreadPoolBoundHandle.Windows.cs" />
+    <Compile Include="$(BclSourcesRoot)\System\Threading\ThreadPool.CoreCLR.Windows.cs" />
   </ItemGroup>
   <ItemGroup Condition="'$(FeatureObjCMarshal)' == 'true'">
     <Compile Include="$(BclSourcesRoot)\System\Runtime\InteropServices\ObjectiveCMarshal.CoreCLR.cs" />

src/coreclr/System.Private.CoreLib/src/System/Threading/Overlapped.cs

@@ -27,28 +27,8 @@ namespace System.Threading
 {
     #region class _IOCompletionCallback
 
-    internal unsafe class _IOCompletionCallback
+    internal unsafe partial class _IOCompletionCallback
     {
-        private readonly IOCompletionCallback _ioCompletionCallback;
-        private readonly ExecutionContext _executionContext;
-        private uint _errorCode; // Error code
-        private uint _numBytes; // No. of bytes transferred
-        private NativeOverlapped* _pNativeOverlapped;
-
-        internal _IOCompletionCallback(IOCompletionCallback ioCompletionCallback, ExecutionContext executionContext)
-        {
-            _ioCompletionCallback = ioCompletionCallback;
-            _executionContext = executionContext;
-        }
-        // Context callback: same sig for SendOrPostCallback and ContextCallback
-        internal static ContextCallback _ccb = new ContextCallback(IOCompletionCallback_Context);
-        internal static void IOCompletionCallback_Context(object? state)
-        {
-            _IOCompletionCallback? helper = (_IOCompletionCallback?)state;
-            Debug.Assert(helper != null, "_IOCompletionCallback cannot be null");
-            helper._ioCompletionCallback(helper._errorCode, helper._numBytes, helper._pNativeOverlapped);
-        }
-
         // call back helper
         internal static void PerformIOCompletionCallback(uint errorCode, uint numBytes, NativeOverlapped* pNativeOverlapped)
         {
@@ -69,7 +49,7 @@ internal static void PerformIOCompletionCallback(uint errorCode, uint numBytes,
                     helper._errorCode = errorCode;
                     helper._numBytes = numBytes;
                     helper._pNativeOverlapped = pNativeOverlapped;
-                    ExecutionContext.RunInternal(helper._executionContext, _ccb, helper);
+                    ExecutionContext.RunInternal(helper._executionContext, IOCompletionCallback_Context_Delegate, helper);
                 }
 
                 // Quickly check the VM again, to see if a packet has arrived.
@@ -132,8 +112,6 @@ internal sealed unsafe class OverlappedData
             return AllocateNativeOverlapped();
         }
 
-        internal bool IsUserObject(byte[]? buffer) => ReferenceEquals(_userObject, buffer);
-
         [MethodImpl(MethodImplOptions.InternalCall)]
         private extern NativeOverlapped* AllocateNativeOverlapped();
 
@@ -254,8 +232,6 @@ public static unsafe void Free(NativeOverlapped* nativeOverlappedPtr!!)
             OverlappedData.GetOverlappedFromNative(nativeOverlappedPtr)._overlapped._overlappedData = null;
             OverlappedData.FreeNativeOverlapped(nativeOverlappedPtr);
         }
-
-        internal bool IsUserObject(byte[]? buffer) => _overlappedData!.IsUserObject(buffer);
     }
 
     #endregion class Overlapped

src/coreclr/System.Private.CoreLib/src/System/Threading/ThreadPool.CoreCLR.Windows.cs

@@ -0,0 +1,75 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+
+using System.Runtime.CompilerServices;
+using System.Runtime.InteropServices;
+using System.Runtime.Versioning;
+
+namespace System.Threading
+{
+    public static partial class ThreadPool
+    {
+        [CLSCompliant(false)]
+        [SupportedOSPlatform("windows")]
+        public static unsafe bool UnsafeQueueNativeOverlapped(NativeOverlapped* overlapped)
+        {
+            if (overlapped == null)
+            {
+                ThrowHelper.ThrowArgumentNullException(ExceptionArgument.overlapped);
+            }
+
+            if (UsePortableThreadPoolForIO)
+            {
+                // OS doesn't signal handle, so do it here
+                overlapped->InternalLow = IntPtr.Zero;
+
+                PortableThreadPool.ThreadPoolInstance.QueueNativeOverlapped(overlapped);
+                return true;
+            }
+
+            return PostQueuedCompletionStatus(overlapped);
+        }
+
+        [MethodImpl(MethodImplOptions.InternalCall)]
+        private static extern unsafe bool PostQueuedCompletionStatus(NativeOverlapped* overlapped);
+
+        [Obsolete("ThreadPool.BindHandle(IntPtr) has been deprecated. Use ThreadPool.BindHandle(SafeHandle) instead.")]
+        [SupportedOSPlatform("windows")]
+        public static bool BindHandle(IntPtr osHandle)
+        {
+            if (UsePortableThreadPoolForIO)
+            {
+                PortableThreadPool.ThreadPoolInstance.RegisterForIOCompletionNotifications(osHandle);
+                return true;
+            }
+
+            return BindIOCompletionCallbackNative(osHandle);
+        }
+
+        [SupportedOSPlatform("windows")]
+        public static bool BindHandle(SafeHandle osHandle!!)
+        {
+            bool mustReleaseSafeHandle = false;
+            try
+            {
+                osHandle.DangerousAddRef(ref mustReleaseSafeHandle);
+
+                if (UsePortableThreadPoolForIO)
+                {
+                    PortableThreadPool.ThreadPoolInstance.RegisterForIOCompletionNotifications(osHandle.DangerousGetHandle());
+                    return true;
+                }
+
+                return BindIOCompletionCallbackNative(osHandle.DangerousGetHandle());
+            }
+            finally
+            {
+                if (mustReleaseSafeHandle)
+                    osHandle.DangerousRelease();
+            }
+        }
+
+        [MethodImpl(MethodImplOptions.InternalCall)]
+        private static extern bool BindIOCompletionCallbackNative(IntPtr fileHandle);
+    }
+}