Move FlushProcessWriteBuffers to minipal (#118161)

* Move FlushProcessWriteBuffers to minipal

Implements #117977

Also move the initialization of it and get of page size

* Fix build

* Fix linux build

* Update definitions

Co-authored-by: Adeel Mujahid <3840695+am11@users.noreply.github.com>

* Feedback

Do not care about cleanup

* Feedback

Do not move page size to minipal

* Feedback

Rename the files

* Feedback

Changes which were missed in earlier commits

* Feedback

Remove PalFlushProcessWriteBuffers wrapper

* Feedback

Remove parts around g_helperPage on mac

* Feedback

Rename to memory_barrier_process_wide

* Fix mac build

Co-authored-by: Adeel Mujahid <3840695+am11@users.noreply.github.com>

* Fix build

* Fix windows build

* More specific include

Co-authored-by: Adeel Mujahid <3840695+am11@users.noreply.github.com>

* Fix windows build

Co-authored-by: Adeel Mujahid <3840695+am11@users.noreply.github.com>

* Feedback

Remove windows wrapper too

* Use more generic include again

Co-authored-by: Adeel Mujahid <3840695+am11@users.noreply.github.com>

* Apply suggestions from code review

Refactor ifdefs

Co-authored-by: Jan Kotas <jkotas@microsoft.com>

* Fix build

* Complete the suggestion from feedback

* Feedback

Call `minipal_initialize_memory_barrier_process_wide` from unix
`GCToOSInterface::Initialize`. Let it return true if it was previously
successfully initialized.

* Fix windows build

* Apply suggestions from code review

Co-authored-by: Jan Kotas <jkotas@microsoft.com>

* Feedback

Refactor minipal_memory_barrier_process_wide

* Fix build

* Apply suggestions from code review

---------

Co-authored-by: Adeel Mujahid <3840695+am11@users.noreply.github.com>
Co-authored-by: Jan Kotas <jkotas@microsoft.com>

Author: 3 people

src/coreclr/gc/env/gcenv.os.h

@@ -451,9 +451,6 @@ class GCToOSInterface
     // Misc
     //
 
-    // Flush write buffers of processors that are executing threads of the current process
-    static void FlushProcessWriteBuffers();
-
     // Break into a debugger
     static void DebugBreak();
 

src/coreclr/gc/gc.cpp

@@ -27,6 +27,7 @@
 #include "handletable.inl"
 #include "gcenv.inl"
 #include "gceventstatus.h"
+#include <minipal/memorybarrierprocesswide.h>
 
 #ifdef __INTELLISENSE__
 #if defined(FEATURE_SVR_GC)
@@ -9871,7 +9872,7 @@ int gc_heap::grow_brick_card_tables (uint8_t* start,
         if (!write_barrier_updated)
         {
             seg_mapping_table = new_seg_mapping_table;
-            GCToOSInterface::FlushProcessWriteBuffers();
+            minipal_memory_barrier_process_wide();
             g_gc_lowest_address = saved_g_lowest_address;
             g_gc_highest_address = saved_g_highest_address;
 

src/coreclr/gc/softwarewritewatch.cpp

@@ -5,6 +5,7 @@
 #include "gcenv.h"
 #include "env/gcenv.os.h"
 #include "softwarewritewatch.h"
+#include <minipal/memorybarrierprocesswide.h>
 
 #ifdef FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
 #ifndef DACCESS_COMPILE
@@ -131,7 +132,7 @@ void SoftwareWriteWatch::GetDirty(
     {
         // When a page is marked as dirty, a memory barrier is not issued after the write most of the time. Issue a memory
         // barrier on all active threads of the process now to make recent changes to dirty state visible to this thread.
-        GCToOSInterface::FlushProcessWriteBuffers();
+        minipal_memory_barrier_process_wide();
     }
 
     uint8_t *tableRegionStart;
@@ -233,7 +234,7 @@ void SoftwareWriteWatch::GetDirty(
         // that the GC will not miss marking through dirtied objects in the page. Issue a memory barrier on all active threads
         // of the process now.
         MemoryBarrier(); // flush writes from this thread first to guarantee ordering
-        GCToOSInterface::FlushProcessWriteBuffers();
+        minipal_memory_barrier_process_wide();
     }
 }
 

src/coreclr/gc/unix/gcenv.unix.cpp

@@ -23,24 +23,14 @@
 #include "volatile.h"
 #include "gcconfig.h"
 #include "numasupport.h"
+#include <minipal/memorybarrierprocesswide.h>
 #include <minipal/thread.h>
 #include <minipal/time.h>
 
 #if HAVE_SWAPCTL
 #include <sys/swap.h>
 #endif
 
-#ifdef __linux__
-#include <linux/membarrier.h>
-#include <sys/syscall.h>
-#define membarrier(...) syscall(__NR_membarrier, __VA_ARGS__)
-#elif HAVE_SYS_MEMBARRIER_H
-#include <sys/membarrier.h>
-#ifdef TARGET_BROWSER
-#define membarrier(cmd, flags, cpu_id) 0 // browser/wasm is currently single threaded
-#endif
-#endif
-
 #include <sys/resource.h>
 
 #undef min
@@ -85,21 +75,6 @@
 
 #include <mach/task.h>
 #include <mach/vm_map.h>
-extern "C"
-{
-#  include <mach/thread_state.h>
-}
-
-#define CHECK_MACH(_msg, machret) do {                                      \
-        if (machret != KERN_SUCCESS)                                        \
-        {                                                                   \
-            char _szError[1024];                                            \
-            snprintf(_szError, ARRAY_SIZE(_szError), "%s: %u: %s", __FUNCTION__, __LINE__, _msg);  \
-            mach_error(_szError, machret);                                  \
-            abort();                                                        \
-        }                                                                   \
-    } while (false)
-
 #endif // __APPLE__
 
 #ifdef __HAIKU__
@@ -140,48 +115,6 @@ typedef cpuset_t cpu_set_t;
 // The cached total number of CPUs that can be used in the OS.
 static uint32_t g_totalCpuCount = 0;
 
-bool CanFlushUsingMembarrier()
-{
-#if defined(__linux__) || HAVE_SYS_MEMBARRIER_H
-
-#ifdef TARGET_ANDROID
-    // Avoid calling membarrier on older Android versions where membarrier
-    // may be barred by seccomp causing the process to be killed.
-    int apiLevel = android_get_device_api_level();
-    if (apiLevel < __ANDROID_API_Q__)
-    {
-        return false;
-    }
-#endif
-
-    // Starting with Linux kernel 4.14, process memory barriers can be generated
-    // using MEMBARRIER_CMD_PRIVATE_EXPEDITED.
-
-    int mask = membarrier(MEMBARRIER_CMD_QUERY, 0, 0);
-
-    if (mask >= 0 &&
-        mask & MEMBARRIER_CMD_PRIVATE_EXPEDITED &&
-        // Register intent to use the private expedited command.
-        membarrier(MEMBARRIER_CMD_REGISTER_PRIVATE_EXPEDITED, 0, 0) == 0)
-    {
-        return true;
-    }
-#endif
-
-    return false;
-}
-
-//
-// Tracks if the OS supports FlushProcessWriteBuffers using membarrier
-//
-static int s_flushUsingMemBarrier = 0;
-
-// Helper memory page used by the FlushProcessWriteBuffers
-static uint8_t* g_helperPage = 0;
-
-// Mutex to make the FlushProcessWriteBuffersMutex thread safe
-static pthread_mutex_t g_flushProcessWriteBuffersMutex;
-
 size_t GetRestrictedPhysicalMemoryLimit();
 bool GetPhysicalMemoryUsed(size_t* val);
 
@@ -219,50 +152,10 @@ bool GCToOSInterface::Initialize()
 
     g_totalCpuCount = cpuCount;
 
-    //
-    // support for FlusProcessWriteBuffers
-    //
-#ifndef TARGET_WASM
-    assert(s_flushUsingMemBarrier == 0);
-
-    if (CanFlushUsingMembarrier())
-    {
-        s_flushUsingMemBarrier = TRUE;
-    }
-#ifndef TARGET_APPLE
-    else
+    if (!minipal_initialize_memory_barrier_process_wide())
     {
-        assert(g_helperPage == 0);
-
-        g_helperPage = static_cast<uint8_t*>(mmap(0, OS_PAGE_SIZE, PROT_READ | PROT_WRITE, MAP_ANONYMOUS | MAP_PRIVATE, -1, 0));
-
-        if (g_helperPage == MAP_FAILED)
-        {
-            return false;
-        }
-
-        // Verify that the s_helperPage is really aligned to the g_SystemInfo.dwPageSize
-        assert((((size_t)g_helperPage) & (OS_PAGE_SIZE - 1)) == 0);
-
-        // Locking the page ensures that it stays in memory during the two mprotect
-        // calls in the FlushProcessWriteBuffers below. If the page was unmapped between
-        // those calls, they would not have the expected effect of generating IPI.
-        int status = mlock(g_helperPage, OS_PAGE_SIZE);
-
-        if (status != 0)
-        {
-            return false;
-        }
-
-        status = pthread_mutex_init(&g_flushProcessWriteBuffersMutex, NULL);
-        if (status != 0)
-        {
-            munlock(g_helperPage, OS_PAGE_SIZE);
-            return false;
-        }
+        return false;
     }
-#endif // !TARGET_APPLE
-#endif // !TARGET_WASM
 
     InitializeCGroup();
 
@@ -354,13 +247,6 @@ bool GCToOSInterface::Initialize()
 // Shutdown the interface implementation
 void GCToOSInterface::Shutdown()
 {
-    int ret = munlock(g_helperPage, OS_PAGE_SIZE);
-    assert(ret == 0);
-    ret = pthread_mutex_destroy(&g_flushProcessWriteBuffersMutex);
-    assert(ret == 0);
-
-    munmap(g_helperPage, OS_PAGE_SIZE);
-
     CleanupCGroup();
 }
 
@@ -410,91 +296,6 @@ bool GCToOSInterface::CanGetCurrentProcessorNumber()
     return HAVE_SCHED_GETCPU;
 }
 
-// Flush write buffers of processors that are executing threads of the current process
-void GCToOSInterface::FlushProcessWriteBuffers()
-{
-#ifndef TARGET_WASM
-#if defined(__linux__) || HAVE_SYS_MEMBARRIER_H
-    if (s_flushUsingMemBarrier)
-    {
-        int status = membarrier(MEMBARRIER_CMD_PRIVATE_EXPEDITED, 0, 0);
-        assert(status == 0 && "Failed to flush using membarrier");
-    }
-    else
-#endif
-    if (g_helperPage != 0)
-    {
-        int status = pthread_mutex_lock(&g_flushProcessWriteBuffersMutex);
-        assert(status == 0 && "Failed to lock the flushProcessWriteBuffersMutex lock");
-
-        // Changing a helper memory page protection from read / write to no access
-        // causes the OS to issue IPI to flush TLBs on all processors. This also
-        // results in flushing the processor buffers.
-        status = mprotect(g_helperPage, OS_PAGE_SIZE, PROT_READ | PROT_WRITE);
-        assert(status == 0 && "Failed to change helper page protection to read / write");
-
-        // Ensure that the page is dirty before we change the protection so that
-        // we prevent the OS from skipping the global TLB flush.
-        __sync_add_and_fetch((size_t*)g_helperPage, 1);
-
-        status = mprotect(g_helperPage, OS_PAGE_SIZE, PROT_NONE);
-        assert(status == 0 && "Failed to change helper page protection to no access");
-
-        status = pthread_mutex_unlock(&g_flushProcessWriteBuffersMutex);
-        assert(status == 0 && "Failed to unlock the flushProcessWriteBuffersMutex lock");
-    }
-#ifdef TARGET_APPLE
-    else
-    {
-        mach_msg_type_number_t cThreads;
-        thread_act_t *pThreads;
-        kern_return_t machret = task_threads(mach_task_self(), &pThreads, &cThreads);
-        CHECK_MACH("task_threads()", machret);
-
-        uintptr_t sp;
-        uintptr_t registerValues[128];
-
-        // Iterate through each of the threads in the list.
-        for (mach_msg_type_number_t i = 0; i < cThreads; i++)
-        {
-            if (__builtin_available (macOS 10.14, iOS 12, tvOS 9, *))
-            {
-                // Request the threads pointer values to force the thread to emit a memory barrier
-                size_t registers = 128;
-                machret = thread_get_register_pointer_values(pThreads[i], &sp, &registers, registerValues);
-            }
-            else
-            {
-                // fallback implementation for older OS versions
-#if defined(HOST_AMD64)
-                x86_thread_state64_t threadState;
-                mach_msg_type_number_t count = x86_THREAD_STATE64_COUNT;
-                machret = thread_get_state(pThreads[i], x86_THREAD_STATE64, (thread_state_t)&threadState, &count);
-#elif defined(HOST_ARM64)
-                arm_thread_state64_t threadState;
-                mach_msg_type_number_t count = ARM_THREAD_STATE64_COUNT;
-                machret = thread_get_state(pThreads[i], ARM_THREAD_STATE64, (thread_state_t)&threadState, &count);
-#else
-                #error Unexpected architecture
-#endif
-            }
-
-            if (machret == KERN_INSUFFICIENT_BUFFER_SIZE)
-            {
-                CHECK_MACH("thread_get_register_pointer_values()", machret);
-            }
-
-            machret = mach_port_deallocate(mach_task_self(), pThreads[i]);
-            CHECK_MACH("mach_port_deallocate()", machret);
-        }
-        // Deallocate the thread list now we're done with it.
-        machret = vm_deallocate(mach_task_self(), (vm_address_t)pThreads, cThreads * sizeof(thread_act_t));
-        CHECK_MACH("vm_deallocate()", machret);
-    }
-#endif // TARGET_APPLE
-#endif // !TARGET_WASM
-}
-
 // Break into a debugger. Uses a compiler intrinsic if one is available,
 // otherwise raises a SIGTRAP.
 void GCToOSInterface::DebugBreak()

src/coreclr/gc/windows/gcenv.windows.cpp

@@ -640,12 +640,6 @@ bool GCToOSInterface::CanGetCurrentProcessorNumber()
     return true;
 }
 
-// Flush write buffers of processors that are executing threads of the current process
-void GCToOSInterface::FlushProcessWriteBuffers()
-{
-    ::FlushProcessWriteBuffers();
-}
-
 // Break into a debugger
 void GCToOSInterface::DebugBreak()
 {

src/coreclr/nativeaot/Runtime/MiscHelpers.cpp

@@ -73,7 +73,7 @@ EXTERN_C void QCALLTYPE RhFlushProcessWriteBuffers()
     ASSERT_MSG(!ThreadStore::GetCurrentThread()->IsCurrentThreadInCooperativeMode(),
         "You must p/invoke to RhFlushProcessWriteBuffers");
 
-    PalFlushProcessWriteBuffers();
+    minipal_memory_barrier_process_wide();
 }
 
 // Get the list of currently loaded NativeAOT modules (as OS HMODULE handles). The caller provides a reference

src/coreclr/nativeaot/Runtime/Pal.h

@@ -23,6 +23,7 @@
 #include <pthread.h>
 #endif
 
+#include <minipal/memorybarrierprocesswide.h>
 #include <minipal/guid.h>
 
 #include "CommonTypes.h"
@@ -290,7 +291,6 @@ int32_t _stricmp(const char *string1, const char *string2);
 
 uint16_t PalCaptureStackBackTrace(uint32_t arg1, uint32_t arg2, void* arg3, uint32_t* arg4);
 UInt32_BOOL PalCloseHandle(HANDLE arg1);
-void PalFlushProcessWriteBuffers();
 uint32_t PalGetCurrentProcessId();
 
 #ifdef UNICODE

src/coreclr/nativeaot/Runtime/gcenv.ee.cpp

@@ -63,7 +63,7 @@ void GCToEEInterface::RestartEE(bool /*bFinishedGC*/)
     // This is needed to synchronize threads that were running in preemptive mode while
     // the runtime was suspended and that will return to cooperative mode after the runtime
     // is restarted.
-    PalFlushProcessWriteBuffers();
+    minipal_memory_barrier_process_wide();
 #endif // !defined(TARGET_X86) && !defined(TARGET_AMD64)
 
     SyncClean::CleanUp();
@@ -404,7 +404,7 @@ void GCToEEInterface::StompWriteBarrier(WriteBarrierParameters* args)
         {
             // If runtime is not suspended, force all threads to see the changed table before seeing updated heap boundaries.
             // See: http://vstfdevdiv:8080/DevDiv2/DevDiv/_workitems/edit/346765
-            PalFlushProcessWriteBuffers();
+            minipal_memory_barrier_process_wide();
         }
 #endif
 
@@ -415,7 +415,7 @@ void GCToEEInterface::StompWriteBarrier(WriteBarrierParameters* args)
         if (!is_runtime_suspended)
         {
             // If runtime is not suspended, force all threads to see the changed state before observing future allocations.
-            PalFlushProcessWriteBuffers();
+            minipal_memory_barrier_process_wide();
         }
 #endif
         return;