[DRAFT] Write barrier without any RWX pages

src/coreclr/inc/clrconfigvalues.h

@@ -254,6 +254,7 @@ CONFIG_DWORD_INFO(INTERNAL_FastGCCheckStack, W("FastGCCheckStack"), 0, "")
 CONFIG_DWORD_INFO(INTERNAL_FastGCStress, W("FastGCStress"), 0, "Reduce the number of GCs done by enabling GCStress")
 RETAIL_CONFIG_DWORD_INFO(UNSUPPORTED_GCBreakOnOOM, W("GCBreakOnOOM"), 0, "Does a DebugBreak at the soonest time we detect an OOM")
 RETAIL_CONFIG_DWORD_INFO(UNSUPPORTED_gcConcurrent, W("gcConcurrent"), (DWORD)-1, "Enables/Disables concurrent GC")
+RETAIL_CONFIG_DWORD_INFO(UNSUPPORTED_UseGCWriteBarrierCopy, W("UseGCWriteBarrierCopy"), 0, "Use a copy of the write barrier for the GC. This is somewhat faster and for optimizations where the barrier is mutated as the program runs.")
 
 #ifdef FEATURE_CONSERVATIVE_GC
 RETAIL_CONFIG_DWORD_INFO(UNSUPPORTED_gcConservative, W("gcConservative"), 0, "Enables/Disables conservative GC")

src/coreclr/inc/jithelpers.h

@@ -183,11 +183,11 @@
     JITHELPER(CORINFO_HELP_CHECK_OBJ,           JIT_CheckObj,       METHOD__NIL)
 
     // GC Write barrier support
-    DYNAMICJITHELPER(CORINFO_HELP_ASSIGN_REF,   JIT_WriteBarrier,   METHOD__NIL)
-    DYNAMICJITHELPER(CORINFO_HELP_CHECKED_ASSIGN_REF, JIT_CheckedWriteBarrier,METHOD__NIL)
+    DYNAMICJITHELPER(CORINFO_HELP_ASSIGN_REF,   RhpAssignRef,   METHOD__NIL)
+    DYNAMICJITHELPER(CORINFO_HELP_CHECKED_ASSIGN_REF, RhpCheckedAssignRef,METHOD__NIL)
     JITHELPER(CORINFO_HELP_ASSIGN_REF_ENSURE_NONHEAP, JIT_WriteBarrierEnsureNonHeapTarget,METHOD__NIL)
 
-    DYNAMICJITHELPER(CORINFO_HELP_ASSIGN_BYREF, JIT_ByRefWriteBarrier,METHOD__NIL)
+    DYNAMICJITHELPER(CORINFO_HELP_ASSIGN_BYREF, RhpByRefAssignRef,METHOD__NIL)
     DYNAMICJITHELPER(CORINFO_HELP_BULK_WRITEBARRIER, NULL, METHOD__BUFFER__MEMCOPYGC)
 
     // Accessing fields
@@ -285,19 +285,19 @@
 #endif // !FEATURE_EH_FUNCLETS
 
 #ifdef TARGET_X86
-    DYNAMICJITHELPER(CORINFO_HELP_ASSIGN_REF_EAX, JIT_WriteBarrierEAX, METHOD__NIL)
-    DYNAMICJITHELPER(CORINFO_HELP_ASSIGN_REF_EBX, JIT_WriteBarrierEBX, METHOD__NIL)
-    DYNAMICJITHELPER(CORINFO_HELP_ASSIGN_REF_ECX, JIT_WriteBarrierECX, METHOD__NIL)
-    DYNAMICJITHELPER(CORINFO_HELP_ASSIGN_REF_ESI, JIT_WriteBarrierESI, METHOD__NIL)
-    DYNAMICJITHELPER(CORINFO_HELP_ASSIGN_REF_EDI, JIT_WriteBarrierEDI, METHOD__NIL)
-    DYNAMICJITHELPER(CORINFO_HELP_ASSIGN_REF_EBP, JIT_WriteBarrierEBP, METHOD__NIL)
-
-    JITHELPER(CORINFO_HELP_CHECKED_ASSIGN_REF_EAX, JIT_CheckedWriteBarrierEAX, METHOD__NIL)
-    JITHELPER(CORINFO_HELP_CHECKED_ASSIGN_REF_EBX, JIT_CheckedWriteBarrierEBX, METHOD__NIL)
-    JITHELPER(CORINFO_HELP_CHECKED_ASSIGN_REF_ECX, JIT_CheckedWriteBarrierECX, METHOD__NIL)
-    JITHELPER(CORINFO_HELP_CHECKED_ASSIGN_REF_ESI, JIT_CheckedWriteBarrierESI, METHOD__NIL)
-    JITHELPER(CORINFO_HELP_CHECKED_ASSIGN_REF_EDI, JIT_CheckedWriteBarrierEDI, METHOD__NIL)
-    JITHELPER(CORINFO_HELP_CHECKED_ASSIGN_REF_EBP, JIT_CheckedWriteBarrierEBP, METHOD__NIL)
+    DYNAMICJITHELPER(CORINFO_HELP_ASSIGN_REF_EAX, RhpAssignRefEAX, METHOD__NIL)
+    DYNAMICJITHELPER(CORINFO_HELP_ASSIGN_REF_EBX, RhpAssignRefEBX, METHOD__NIL)
+    DYNAMICJITHELPER(CORINFO_HELP_ASSIGN_REF_ECX, RhpAssignRefECX, METHOD__NIL)
+    DYNAMICJITHELPER(CORINFO_HELP_ASSIGN_REF_ESI, RhpAssignRefESI, METHOD__NIL)
+    DYNAMICJITHELPER(CORINFO_HELP_ASSIGN_REF_EDI, RhpAssignRefEDI, METHOD__NIL)
+    DYNAMICJITHELPER(CORINFO_HELP_ASSIGN_REF_EBP, RhpAssignRefEBP, METHOD__NIL)
+
+    JITHELPER(CORINFO_HELP_CHECKED_ASSIGN_REF_EAX, RhpCheckedAssignRefEAX, METHOD__NIL)
+    JITHELPER(CORINFO_HELP_CHECKED_ASSIGN_REF_EBX, RhpCheckedAssignRefEBX, METHOD__NIL)
+    JITHELPER(CORINFO_HELP_CHECKED_ASSIGN_REF_ECX, RhpCheckedAssignRefECX, METHOD__NIL)
+    JITHELPER(CORINFO_HELP_CHECKED_ASSIGN_REF_ESI, RhpCheckedAssignRefESI, METHOD__NIL)
+    JITHELPER(CORINFO_HELP_CHECKED_ASSIGN_REF_EDI, RhpCheckedAssignRefEDI, METHOD__NIL)
+    JITHELPER(CORINFO_HELP_CHECKED_ASSIGN_REF_EBP, RhpCheckedAssignRefEBP, METHOD__NIL)
 #else
     JITHELPER(CORINFO_HELP_ASSIGN_REF_EAX, NULL, METHOD__NIL)
     JITHELPER(CORINFO_HELP_ASSIGN_REF_EBX, NULL, METHOD__NIL)

src/coreclr/nativeaot/Runtime/CMakeLists.txt

@@ -213,7 +213,7 @@ list(APPEND RUNTIME_SOURCES_ARCH_ASM
   ${ARCH_SOURCES_DIR}/InteropThunksHelpers.${ASM_SUFFIX}
   ${RUNTIME_DIR}/${ARCH_SOURCES_DIR}/StubDispatch.${ASM_SUFFIX}
   ${ARCH_SOURCES_DIR}/UniversalTransition.${ASM_SUFFIX}
-  ${ARCH_SOURCES_DIR}/WriteBarriers.${ASM_SUFFIX}
+  ${RUNTIME_DIR}/${ARCH_SOURCES_DIR}/WriteBarriers.${ASM_SUFFIX}
 )
 
 if (CLR_CMAKE_TARGET_ARCH_AMD64 OR CLR_CMAKE_TARGET_ARCH_ARM64)

src/coreclr/nativeaot/Runtime/arm/AsmMacros_Shared.h

@@ -0,0 +1,7 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+
+// This file is used to allow sharing of assembly code between NativeAOT and CoreCLR, which have different conventions about how to ensure that constants offsets are accessible
+
+#include "AsmOffsets.inc"
+#include <unixasmmacros.inc>

src/coreclr/nativeaot/Runtime/i386/AsmMacros_Shared.inc

@@ -0,0 +1,6 @@
+; Licensed to the .NET Foundation under one or more agreements.
+; The .NET Foundation licenses this file to you under the MIT license.
+
+; This file is used to allow sharing of assembly code between NativeAOT and CoreCLR, which have different conventions about how to ensure that constants offsets are accessible
+
+include AsmMacros.inc

src/coreclr/nativeaot/Runtime/loongarch64/AsmMacros_Shared.h

@@ -0,0 +1,7 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+
+// This file is used to allow sharing of assembly code between NativeAOT and CoreCLR, which have different conventions about how to ensure that constants offsets are accessible
+
+#include <unixasmmacros.inc>
+#include "AsmOffsets.inc"

src/coreclr/nativeaot/Runtime/riscv64/AsmMacros_Shared.h

@@ -0,0 +1,7 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+
+// This file is used to allow sharing of assembly code between NativeAOT and CoreCLR, which have different conventions about how to ensure that constants offsets are accessible
+
+#include <unixasmmacros.inc>
+#include "AsmOffsets.inc"

src/coreclr/pal/inc/unixasmmacrosarm.inc

@@ -44,10 +44,28 @@ C_FUNC(\Name\()_End):
         nop
 .endm
 
+.macro GLOBAL_LABEL Name
+        .global C_FUNC(\Name)
+C_FUNC(\Name):
+.endm
+
+.macro ALTERNATE_ENTRY Name
+        .global C_FUNC(\Name)
+        .type \Name, %function
+C_FUNC(\Name):
+.endm
+
 .macro PREPARE_EXTERNAL_VAR Name, HelperReg
         ldr \HelperReg, [pc, #C_FUNC(\Name)@GOTPCREL]
 .endm
 
+.macro PREPARE_EXTERNAL_VAR_INDIRECT Name, HelperReg
+        movw \HelperReg, #:lower16:C_FUNC(\Name) - (. + 12)
+        movt \HelperReg, #:upper16:C_FUNC(\Name) - (. + 8)
+        add \HelperReg, pc
+        ldr \HelperReg, [\HelperReg]
+.endm
+
 .macro push_nonvol_reg Register
         push \Register
         .save \Register

src/coreclr/pal/inc/unixasmmacrosloongarch64.inc

@@ -17,6 +17,12 @@
 C_FUNC(\Name):
 .endm
 
+.macro ALTERNATE_ENTRY Name
+        .global C_FUNC(\Name)
+        .hidden C_FUNC(\Name)
+C_FUNC(\Name):
+.endm
+
 .macro LEAF_ENTRY Name, Section
         .global C_FUNC(\Name)
         .type \Name, %function
@@ -41,6 +47,16 @@ C_FUNC(\Name\()_End):
     la.local  \HelperReg, \Name
 .endm
 
+.macro PREPARE_EXTERNAL_VAR_INDIRECT Name, HelperReg
+        la.local  \HelperReg, \Name
+        ld.d  \HelperReg, \HelperReg, 0
+.endm
+
+.macro PREPARE_EXTERNAL_VAR_INDIRECT_W Name, HelperReg
+        la.local  \HelperReg, \Name
+        ld.w  \HelperReg, \HelperReg, 0
+.endm
+
 .macro PROLOG_STACK_ALLOC Size
         addi.d $sp, $sp, -\Size
         //.cfi_adjust_cfa_offset \Size

src/coreclr/pal/inc/unixasmmacrosriscv64.inc

@@ -37,6 +37,12 @@ C_FUNC(\Name\()_End):
     nop
 .endm
 
+.macro ALTERNATE_ENTRY Name
+        .global C_FUNC(\Name)
+        .hidden C_FUNC(\Name)
+C_FUNC(\Name):
+.endm
+
 .macro PREPARE_EXTERNAL_VAR Name, HelperReg
     lla  \HelperReg, \Name
 .endm

src/coreclr/nativeaot/Runtime/amd64/WriteBarriers.S → src/coreclr/runtime/amd64/WriteBarriers.S

@@ -2,7 +2,7 @@
 // The .NET Foundation licenses this file to you under the MIT license.
 
 .intel_syntax noprefix
-#include <unixasmmacros.inc>
+#include "AsmMacros_Shared.h"
 
 #ifdef WRITE_BARRIER_CHECK
 

src/coreclr/nativeaot/Runtime/amd64/WriteBarriers.asm → src/coreclr/runtime/amd64/WriteBarriers.asm

@@ -1,7 +1,7 @@
 ;; Licensed to the .NET Foundation under one or more agreements.
 ;; The .NET Foundation licenses this file to you under the MIT license.
 
-include AsmMacros.inc
+include AsmMacros_Shared.inc
 
 ;; Macro used to copy contents of newly updated GC heap locations to a shadow copy of the heap. This is used
 ;; during garbage collections to verify that object references where never written to the heap without using a

src/coreclr/runtime/arm/StubDispatch.S

@@ -4,8 +4,7 @@
 .syntax unified
 .thumb
 
-#include <AsmOffsets.inc>         // generated by the build from AsmOffsets.cpp
-#include <unixasmmacros.inc>
+#include "AsmMacros_Shared.h"
 
 #ifdef FEATURE_CACHED_INTERFACE_DISPATCH
 

src/coreclr/nativeaot/Runtime/arm/WriteBarriers.S → src/coreclr/runtime/arm/WriteBarriers.S

@@ -4,79 +4,12 @@
 .syntax unified
 .thumb
 
-#include <AsmOffsets.inc>         // generated by the build from AsmOffsets.cpp
-#include <unixasmmacros.inc>
+#include "AsmMacros_Shared.h"
 
 #ifdef WRITE_BARRIER_CHECK
 
 .macro UPDATE_GC_SHADOW BASENAME, REFREG, DESTREG
-
-          // If g_GCShadow is 0, don't perform the check.
-          PREPARE_EXTERNAL_VAR_INDIRECT g_GCShadow, r12
-          cbz          r12, LOCAL_LABEL(\BASENAME\()_UpdateShadowHeap_Done_\REFREG)
-
-          // Save DESTREG since we're about to modify it (and we need the original value both within the macro and
-          // once we exit the macro). Note that this is naughty since we're altering the stack pointer outside of
-          // the prolog inside a method without a frame. But given that this is only debug code and generally we
-          // shouldn't be walking the stack at this point it seems preferable to recoding the all the barrier
-          // variants to set up frames. The compiler knows exactly which registers are trashed in the simple write
-          // barrier case, so we don't have any more scratch registers to play with (and doing so would only make
-          // things harder if at a later stage we want to allow multiple barrier versions based on the input
-          // registers).
-          push         \DESTREG
-
-          // Transform DESTREG into the equivalent address in the shadow heap.
-          PREPARE_EXTERNAL_VAR_INDIRECT g_lowest_address, r12
-          sub          \DESTREG, r12
-          cmp          \DESTREG, #0
-          blo          LOCAL_LABEL(\BASENAME\()_UpdateShadowHeap_PopThenDone_\REFREG)
-          PREPARE_EXTERNAL_VAR_INDIRECT g_GCShadow, r12
-          add          \DESTREG, r12
-          PREPARE_EXTERNAL_VAR_INDIRECT g_GCShadowEnd, r12
-          cmp          \DESTREG, r12
-          bhs          LOCAL_LABEL(\BASENAME\()_UpdateShadowHeap_PopThenDone_\REFREG)
-
-          // Update the shadow heap.
-          str          \REFREG, [\DESTREG]
-
-          // The following read must be strongly ordered wrt to the write we've just performed in order to
-          // prevent race conditions.
-          dmb
-
-          // Now check that the real heap location still contains the value we just wrote into the shadow heap.
-          mov          r12, \DESTREG
-          ldr          \DESTREG, [sp]
-          str          r12, [sp]
-          ldr          r12, [\DESTREG]
-          cmp          r12, \REFREG
-          bne          LOCAL_LABEL(\BASENAME\()_UpdateShadowHeap_Invalidate_\REFREG)
-
-          // The original DESTREG value is now restored but the stack has a value (the shadow version of the
-          // location) pushed. Need to discard this push before we are done.
-          add          sp, #4
-          b            LOCAL_LABEL(\BASENAME\()_UpdateShadowHeap_Done_\REFREG)
-
-LOCAL_LABEL(\BASENAME\()_UpdateShadowHeap_Invalidate_\REFREG):
-          // Someone went and updated the real heap. We need to invalidate the shadow location since we can't
-          // guarantee whose shadow update won.
-
-          // Retrieve shadow location from the stack and restore original DESTREG to the stack. This is an
-          // additional memory barrier we don't require but it's on the rare path and x86 doesn't have an xchg
-          // variant that doesn't implicitly specify the lock prefix. Note that INVALIDGCVALUE is a 32-bit
-          // immediate and therefore must be moved into a register before it can be written to the shadow
-          // location.
-          mov          r12, \DESTREG
-          ldr          \DESTREG, [sp]
-          str          r12, [sp]
-          push         \REFREG
-          movw         \REFREG, #(INVALIDGCVALUE & 0xFFFF)
-          movt         \REFREG, #(INVALIDGCVALUE >> 16)
-          str          \REFREG, [\DESTREG]
-          pop          \REFREG
-
-LOCAL_LABEL(\BASENAME\()_UpdateShadowHeap_PopThenDone_\REFREG):
-          // Restore original DESTREG value from the stack.
-          pop          \DESTREG
+        // Todo: implement, debugging helper
 
 LOCAL_LABEL(\BASENAME\()_UpdateShadowHeap_Done_\REFREG):
 
@@ -89,6 +22,26 @@ LOCAL_LABEL(\BASENAME\()_UpdateShadowHeap_Done_\REFREG):
 
 #endif // WRITE_BARRIER_CHECK
 
+#ifdef FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
+.macro UPDATE_WRITE_WATCH_TABLE ptrReg, tmpReg
+
+        PREPARE_EXTERNAL_VAR_INDIRECT g_write_watch_table, __wbScratch
+        cbz __wbScratch, 2f
+        add __wbScratch, __wbScratch, \ptrReg, lsr #0xc  // SoftwareWriteWatch::AddressToTableByteIndexShift
+
+        ldrb \tmpReg, [__wbScratch]
+        cmp \tmpReg, #0xff
+        itt ne
+        movne \tmpReg, 0xff
+        strbne \tmpReg, [__wbScratch]
+
+2:
+.endm
+#else
+.macro UPDATE_WRITE_WATCH_TABLE ptrReg, tmpReg
+.endm
+#endif
+
 // There are several different helpers used depending on which register holds the object reference. Since all
 // the helpers have identical structure we use a macro to define this structure. Two arguments are taken, the
 // name of the register that points to the location to be updated and the name of the register that holds the
@@ -109,12 +62,14 @@ LOCAL_LABEL(\BASENAME\()_UpdateShadowHeap_Done_\REFREG):
           cmp          \REFREG, r12
           bhs          LOCAL_LABEL(\BASENAME\()_EXIT_\REFREG)
 
+          UPDATE_WRITE_WATCH_TABLE r0, r12
+
           // We have a location on the GC heap being updated with a reference to an ephemeral object so we must
           // track this write. The location address is translated into an offset in the card table bitmap. We set
           // an entire byte in the card table since it's quicker than messing around with bitmasks and we only write
           // the byte if it hasn't already been done since writes are expensive and impact scaling.
           PREPARE_EXTERNAL_VAR_INDIRECT g_card_table, r12
-          add          r0, r12, r0, lsr #LOG2_CLUMP_SIZE
+          add          r0, r12, r0, lsr #10
           ldrb         r12, [r0]
           cmp          r12, #0x0FF
           bne          LOCAL_LABEL(\BASENAME\()_UpdateCardTable_\REFREG)
@@ -250,6 +205,8 @@ LEAF_END RhpCheckedAssignRef\EXPORT_REG_NAME, _TEXT
 // just one write barrier that assumes the input register is RSI.
 DEFINE_CHECKED_WRITE_BARRIER r1, r1
 
+#ifdef FEATURE_NATIVEAOT
+
 // r0 = destination address
 // r1 = value
 // r2 = comparand
@@ -292,6 +249,7 @@ LOCAL_LABEL(RhpCheckedXchgRetry):
 
           bx           lr
 LEAF_END RhpCheckedXchg, _TEXT
+#endif // FEATURE_NATIVEAOT
 
 //
 // RhpByRefAssignRef simulates movs instruction for object references.
@@ -343,12 +301,14 @@ GLOBAL_LABEL RhpByRefAssignRefAVLocation2
           add          r1, #4
           add          r0, #4
 
+          UPDATE_WRITE_WATCH_TABLE r2, r3
+
           // We have a location on the GC heap being updated with a reference to an ephemeral object so we must
           // track this write. The location address is translated into an offset in the card table bitmap. We set
           // an entire byte in the card table since it's quicker than messing around with bitmasks and we only write
           // the byte if it hasn't already been done since writes are expensive and impact scaling.
           PREPARE_EXTERNAL_VAR_INDIRECT g_card_table, r3
-          add           r2, r3, r2, lsr #LOG2_CLUMP_SIZE
+          add           r2, r3, r2, lsr #10
           ldrb          r3, [r2]
           cmp           r3, #0x0FF
           bne           LOCAL_LABEL(RhpByRefAssignRef_UpdateCardTable)

src/coreclr/nativeaot/Runtime/arm64/WriteBarriers.S → src/coreclr/runtime/arm64/WriteBarriers.S

@@ -1,7 +1,7 @@
 // Licensed to the .NET Foundation under one or more agreements.
 // The .NET Foundation licenses this file to you under the MIT license.
 
-#include <unixasmmacros.inc>
+#include "AsmMacros_Shared.h"
 
 // Macro used to copy contents of newly updated GC heap locations to a shadow copy of the heap. This is used
 // during garbage collections to verify that object references where never written to the heap without using a
@@ -266,6 +266,7 @@ LEAF_ENTRY RhpAssignRef, _TEXT
         b       C_FUNC(RhpAssignRefArm64)
 LEAF_END RhpAssignRef, _TEXT
 
+#ifdef FEATURE_NATIVEAOT
 
 // Interlocked operation helpers where the location is an objectref, thus requiring a GC write barrier upon
 // successful updates.
@@ -395,3 +396,4 @@ LOCAL_LABEL(NoBarrierXchg):
 #ifndef LSE_INSTRUCTIONS_ENABLED_BY_DEFAULT
 .arch_extension nolse
 #endif
+#endif // FEATURE_NATIVEAOT