Use implicit MTA for threadpool threads (dotnet#7367)

* Use implicit MTA for threadpool threads

It is not ok to leave COM initialized on Win32 threadpool threads. This change skips COM initialization
on Win32 threadpool threads completely and takes advantage of implicit MTA that is initialized by the finalizer
thread.

This fix should provide high compatiblity with .NET Framework/Core, without performance overhead of initialization/uninitializing COM every time; or running dedicated threadpool.

Fixes dotnet#7356

Author: jkotas

src/Native/Runtime/FinalizerHelpers.cpp

@@ -130,6 +130,17 @@ void RhEnableFinalization()
     g_FinalizerEvent.Set();
 }
 
+EXTERN_C REDHAWK_API void __cdecl RhInitializeFinalizerThread()
+{
+#ifdef APP_LOCAL_RUNTIME
+    // We may have failed to create the finalizer thread at startup.
+    // Try again now.
+    RhStartFinalizerThread();
+#endif
+
+    g_FinalizerEvent.Set();
+}
+
 EXTERN_C REDHAWK_API void __cdecl RhWaitForPendingFinalizers(UInt32_BOOL allowReentrantWait)
 {
     // This must be called via p/invoke rather than RuntimeImport since it blocks and could starve the GC if
@@ -145,7 +156,7 @@ EXTERN_C REDHAWK_API void __cdecl RhWaitForPendingFinalizers(UInt32_BOOL allowRe
         g_FinalizerEvent.Set();
 
 #ifdef APP_LOCAL_RUNTIME
-        // We may have failed to create the finalizer thread at startup.  
+        // We may have failed to create the finalizer thread at startup.
         // Try again now.
         RhStartFinalizerThread();
 #endif

src/Runtime.Base/src/System/Runtime/__Finalizer.cs

@@ -22,6 +22,10 @@ internal static class __Finalizer
         [NativeCallable(EntryPoint = "ProcessFinalizers", CallingConvention = CallingConvention.Cdecl)]
         public static void ProcessFinalizers()
         {
+#if INPLACE_RUNTIME
+            System.Runtime.FinalizerInitRunner.DoInitialize();
+#endif
+
             while (true)
             {
                 // Wait until there's some work to be done. If true is returned we should finalize objects,

src/System.Private.CoreLib/src/System/Runtime/InitializeFinalizerThread.cs

@@ -15,7 +15,7 @@ public static void DoInitialize()
         {
             // Make sure that the finalizer thread is CoInitialized before any objects are finalized.  If this
             // fails, it will throw an exception and that will go unhandled, triggering a FailFast.
-            Thread.InitializeCom();
+            Thread.InitializeComForFinalizerThread();
         }
     }
 }

src/System.Private.CoreLib/src/System/Runtime/RuntimeImports.cs

@@ -97,6 +97,11 @@ internal static void RhWaitForPendingFinalizers(bool allowReentrantWait)
             RhWaitForPendingFinalizers(allowReentrantWait ? 1 : 0);
         }
 
+#if !PROJECTN
+        [DllImport(RuntimeLibrary, ExactSpelling = true)]
+        internal static extern void RhInitializeFinalizerThread();
+#endif
+
         // Get maximum GC generation number.
         [MethodImplAttribute(MethodImplOptions.InternalCall)]
         [RuntimeImport(RuntimeLibrary, "RhGetMaxGcGeneration")]

src/System.Private.CoreLib/src/System/Threading/Thread.CoreRT.Unix.cs

@@ -137,12 +137,16 @@ private void InitializeComOnNewThread()
         {
         }
 
-        internal static void InitializeCom()
+        internal static void InitializeComForFinalizerThread()
         {
         }
 
         public void DisableComObjectEagerCleanup() { }
-        private static void InitializeExistingThreadPoolThread() { }
+
+        private static void InitializeExistingThreadPoolThread()
+        {
+            ThreadPool.InitializeForThreadPoolThread();
+        }
 
         public void Interrupt() => WaitSubsystem.Interrupt(this);
         internal static void UninterruptibleSleep0() => WaitSubsystem.UninterruptibleSleep0();

src/System.Private.CoreLib/src/System/Threading/Thread.CoreRT.Windows.cs

@@ -21,12 +21,14 @@ public sealed partial class Thread
         private static ApartmentType t_apartmentType;
 
         [ThreadStatic]
-        private static bool t_comInitializedByUs;
+        private static ComState t_comState;
 
         private SafeWaitHandle _osHandle;
 
         private ApartmentState _initialApartmentState = ApartmentState.Unknown;
 
+        private volatile static bool s_comInitializedOnFinalizerThread;
+
         private void PlatformSpecificInitialize()
         {
         }
@@ -269,13 +271,16 @@ public bool TrySetApartmentStateUnchecked(ApartmentState state)
                 }
             }
 
-            if (state != ApartmentState.Unknown)
-            {
-                InitializeCom(state);
-            }
-            else
+            if ((t_comState & ComState.Locked) == 0)
             {
-                UninitializeCom();
+                if (state != ApartmentState.Unknown)
+                {
+                    InitializeCom(state);
+                }
+                else
+                {
+                    UninitializeCom();
+                }
             }
 
             // Clear the cache and check whether new state matches the desired state
@@ -288,9 +293,19 @@ private void InitializeComOnNewThread()
             InitializeCom(_initialApartmentState);
         }
 
-        internal static void InitializeCom(ApartmentState state = ApartmentState.MTA)
+        internal static void InitializeComForFinalizerThread()
         {
-            if (t_comInitializedByUs)
+            InitializeCom();
+
+            // Prevent re-initialization of COM model on finalizer thread
+            t_comState |= ComState.Locked;
+
+            s_comInitializedOnFinalizerThread = true;
+        }
+
+        private static void InitializeCom(ApartmentState state = ApartmentState.MTA)
+        {
+            if ((t_comState & ComState.InitializedByUs) != 0)
                 return;
 
 #if ENABLE_WINRT
@@ -309,7 +324,7 @@ internal static void InitializeCom(ApartmentState state = ApartmentState.MTA)
             if (hr < 0)
                 throw new OutOfMemoryException();
 
-            t_comInitializedByUs = true;
+            t_comState |= ComState.InitializedByUs;
 
             // If the thread has already been CoInitialized to the proper mode, then
             // we don't want to leave an outstanding CoInit so we CoUninit.
@@ -319,23 +334,38 @@ internal static void InitializeCom(ApartmentState state = ApartmentState.MTA)
 
         private static void UninitializeCom()
         {
-            if (!t_comInitializedByUs)
+            if ((t_comState & ComState.InitializedByUs) == 0)
                 return;
 
 #if ENABLE_WINRT
             Interop.WinRT.RoUninitialize();
 #else
             Interop.Ole32.CoUninitialize();
 #endif
-            t_comInitializedByUs = false;
+
+            t_comState &= ~ComState.InitializedByUs;
         }
 
         // TODO: https://github.com/dotnet/corefx/issues/20766
         public void DisableComObjectEagerCleanup() { }
 
         private static void InitializeExistingThreadPoolThread()
         {
+#if PROJECTN
             InitializeCom();
+#else
+            // Take advantage of implicit MTA initialized by the finalizer thread
+            SpinWait sw = new SpinWait();
+            while (!s_comInitializedOnFinalizerThread)
+            {
+                RuntimeImports.RhInitializeFinalizerThread();
+                sw.SpinOnce(0);
+            }
+#endif
+
+            // Prevent re-initialization of COM model on threadpool threads
+            t_comState |= ComState.Locked;
+
             ThreadPool.InitializeForThreadPoolThread();
         }
 
@@ -432,14 +462,21 @@ internal static ApartmentType GetCurrentApartmentType()
             return type;
         }
 
-        internal enum ApartmentType
+        internal enum ApartmentType : byte
         {
             Unknown = 0,
             None,
             STA,
             MTA
         }
 
+        [Flags]
+        internal enum ComState : byte
+        {
+            InitializedByUs = 1,
+            Locked = 2,
+        }
+
         private static int ComputeCurrentProcessorId() => (int)Interop.mincore.GetCurrentProcessorNumber();
     }
 }

src/System.Private.CoreLib/src/System/Threading/ThreadPool.Portable.cs

@@ -330,6 +330,8 @@ internal void WaitForRemoval()
 
     public static partial class ThreadPool
     {
+        internal static void InitializeForThreadPoolThread() { }
+
         public static bool SetMaxThreads(int workerThreads, int completionPortThreads)
         {
             if (workerThreads < 0 || completionPortThreads < 0)

src/Test.CoreLib/src/System/Runtime/InitializeFinalizerThread.cs

@@ -0,0 +1,17 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+// See the LICENSE file in the project root for more information.
+
+using System.Threading;
+
+namespace System.Runtime
+{
+    internal static class FinalizerInitRunner
+    {
+        // Here, we are subscribing to a callback from the runtime.  This callback is made from the finalizer
+        // thread before any objects are finalized.  
+        public static void DoInitialize()
+        {
+        }
+    }
+}

src/Test.CoreLib/src/Test.CoreLib.csproj

@@ -235,6 +235,7 @@
     <Compile Include="System\Runtime\CompilerServices\StaticClassConstructionContext.cs" />
     <Compile Include="System\Runtime\RuntimeImports.cs" />
     <Compile Include="System\Runtime\RuntimeHelpers.cs" />
+    <Compile Include="System\Runtime\InitializeFinalizerThread.cs" />
     <Compile Include="System\Threading\Interlocked.cs" />
     <Compile Include="System\Array.cs" />
     <Compile Include="System\RuntimeExceptionHelpers.cs" />