[NativeAOT] Enable software writewatch and card bundles on Windows. (#77934)

* amd64

* arm64

* fix ARM64 build

* import conditionally

Author: VSadov

src/coreclr/nativeaot/Runtime/CMakeLists.txt

@@ -204,11 +204,8 @@ include_directories(${ARCH_SOURCES_DIR})
 
 add_definitions(-DFEATURE_BASICFREEZE)
 add_definitions(-DFEATURE_CONSERVATIVE_GC)
-
-if(CLR_CMAKE_TARGET_UNIX)
-  add_definitions(-DFEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP)
-  add_definitions(-DFEATURE_MANUALLY_MANAGED_CARD_BUNDLES)
-endif()
+add_definitions(-DFEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP)
+add_definitions(-DFEATURE_MANUALLY_MANAGED_CARD_BUNDLES)
 
 add_definitions(-DFEATURE_CUSTOM_IMPORTS)
 add_definitions(-DFEATURE_DYNAMIC_CODE)

src/coreclr/nativeaot/Runtime/amd64/AsmMacros.inc

@@ -406,10 +406,18 @@ EXTERN REDHAWKGCINTERFACE__STRESSGC             : PROC
 EXTERN THREAD__HIJACKFORGCSTRESS                : PROC
 endif ;; FEATURE_GC_STRESS
 
+EXTERN RhpTrapThreads       : DWORD
 EXTERN g_lowest_address     : QWORD
 EXTERN g_highest_address    : QWORD
 EXTERN g_ephemeral_low      : QWORD
 EXTERN g_ephemeral_high     : QWORD
 EXTERN g_card_table         : QWORD
-EXTERN RhpTrapThreads       : DWORD
+
+ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES
+EXTERN g_card_bundle_table  : QWORD
+endif
+
+ifdef FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
+EXTERN g_write_watch_table  : QWORD
+endif
 

src/coreclr/nativeaot/Runtime/amd64/WriteBarriers.S

@@ -255,21 +255,20 @@ LEAF_END RhpCheckedXchg, _TEXT
 // On entry:
 //      rdi: address of ref-field (assigned to)
 //      rsi: address of the data (source)
-//      rcx: be trashed
 //
 // On exit:
 //      rdi, rsi are incremented by 8,
-//      rcx: trashed
+//      rcx, r10, r11: trashed
 //
 LEAF_ENTRY RhpByRefAssignRef, _TEXT
     mov     rcx, [rsi]
     mov     [rdi], rcx
 
     // Check whether the writes were even into the heap. If not there's no card update required.
     cmp     rdi, [C_VAR(g_lowest_address)]
-    jb      LOCAL_LABEL(RhpByRefAssignRef_NotInHeap)
+    jb      LOCAL_LABEL(RhpByRefAssignRef_NoBarrierRequired)
     cmp     rdi, [C_VAR(g_highest_address)]
-    jae     LOCAL_LABEL(RhpByRefAssignRef_NotInHeap)
+    jae     LOCAL_LABEL(RhpByRefAssignRef_NoBarrierRequired)
 
     // Update the shadow copy of the heap with the same value just written to the same heap. (A no-op unless
     // we're in a debug build and write barrier checking has been enabled).
@@ -293,9 +292,9 @@ LOCAL_LABEL(RhpByRefAssignRef_CheckCardTable):
     // If the reference is to an object that's not in an ephemeral generation we have no need to track it
     // (since the object won't be collected or moved by an ephemeral collection).
     cmp     rcx, [C_VAR(g_ephemeral_low)]
-    jb      LOCAL_LABEL(RhpByRefAssignRef_NotInHeap)
+    jb      LOCAL_LABEL(RhpByRefAssignRef_NoBarrierRequired)
     cmp     rcx, [C_VAR(g_ephemeral_high)]
-    jae     LOCAL_LABEL(RhpByRefAssignRef_NotInHeap)
+    jae     LOCAL_LABEL(RhpByRefAssignRef_NoBarrierRequired)
 
     // move current rdi value into rcx, we need to keep rdi and eventually increment by 8
     mov     rcx, rdi
@@ -307,7 +306,7 @@ LOCAL_LABEL(RhpByRefAssignRef_CheckCardTable):
     shr     rcx, 0x0B
     mov     r10, [C_VAR(g_card_table)]
     cmp     byte ptr [rcx + r10], 0x0FF
-    je      LOCAL_LABEL(RhpByRefAssignRef_NotInHeap)
+    je      LOCAL_LABEL(RhpByRefAssignRef_NoBarrierRequired)
 
 // We get here if it's necessary to update the card table.
     mov     byte ptr [rcx + r10], 0xFF
@@ -317,12 +316,12 @@ LOCAL_LABEL(RhpByRefAssignRef_CheckCardTable):
     shr     rcx, 0x0A
     add     rcx, [C_VAR(g_card_bundle_table)]
     cmp     byte ptr [rcx], 0xFF
-    je      LOCAL_LABEL(RhpByRefAssignRef_NotInHeap)
+    je      LOCAL_LABEL(RhpByRefAssignRef_NoBarrierRequired)
 
     mov     byte ptr [rcx], 0xFF
 #endif
 
-LOCAL_LABEL(RhpByRefAssignRef_NotInHeap):
+LOCAL_LABEL(RhpByRefAssignRef_NoBarrierRequired):
     // Increment the pointers before leaving
     add     rdi, 0x8
     add     rsi, 0x8

src/coreclr/nativeaot/Runtime/amd64/WriteBarriers.asm

@@ -100,6 +100,21 @@ DEFINE_UNCHECKED_WRITE_BARRIER_CORE macro BASENAME, REFREG
     ;; we're in a debug build and write barrier checking has been enabled).
     UPDATE_GC_SHADOW BASENAME, REFREG, rcx
 
+ifdef FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
+    mov     r11, [g_write_watch_table]
+    cmp     r11, 0
+    je      &BASENAME&_CheckCardTable_&REFREG&
+
+    mov     r10, rcx
+    shr     r10, 0Ch ;; SoftwareWriteWatch::AddressToTableByteIndexShift
+    add     r10, r11
+    cmp     byte ptr [r10], 0
+    jne     &BASENAME&_CheckCardTable_&REFREG&
+    mov     byte ptr [r10], 0FFh
+endif
+
+&BASENAME&_CheckCardTable_&REFREG&:
+
     ;; If the reference is to an object that's not in an ephemeral generation we have no need to track it
     ;; (since the object won't be collected or moved by an ephemeral collection).
     cmp     REFREG, [g_ephemeral_low]
@@ -111,17 +126,25 @@ DEFINE_UNCHECKED_WRITE_BARRIER_CORE macro BASENAME, REFREG
     ;; track this write. The location address is translated into an offset in the card table bitmap. We set
     ;; an entire byte in the card table since it's quicker than messing around with bitmasks and we only write
     ;; the byte if it hasn't already been done since writes are expensive and impact scaling.
-    shr     rcx, 11
-    add     rcx, [g_card_table]
+    shr     rcx, 0Bh
+    mov     r10, [g_card_table]
+    cmp     byte ptr [rcx + r10], 0FFh
+    je      &BASENAME&_NoBarrierRequired_&REFREG&
+
+    ;; We get here if it's necessary to update the card table.
+    mov     byte ptr [rcx + r10], 0FFh
+
+ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES
+    ;; Shift rcx by 0Ah more to get the card bundle byte (we shifted by 0x0B already)
+    shr     rcx, 0Ah
+    add     rcx, [g_card_bundle_table]
     cmp     byte ptr [rcx], 0FFh
-    jne     &BASENAME&_UpdateCardTable_&REFREG&
-
-&BASENAME&_NoBarrierRequired_&REFREG&:
-    ret
+    je      &BASENAME&_NoBarrierRequired_&REFREG&
 
-;; We get here if it's necessary to update the card table.
-&BASENAME&_UpdateCardTable_&REFREG&:
     mov     byte ptr [rcx], 0FFh
+endif
+
+&BASENAME&_NoBarrierRequired_&REFREG&:
     ret
 
 endm
@@ -248,55 +271,74 @@ LEAF_END RhpCheckedXchg, _TEXT
 ;; On entry:
 ;;      rdi: address of ref-field (assigned to)
 ;;      rsi: address of the data (source)
-;;      rcx: be trashed
 ;;
 ;; On exit:
 ;;      rdi, rsi are incremented by 8,
-;;      rcx: trashed
+;;      rcx, r10, r11: trashed
 ;;
 LEAF_ENTRY RhpByRefAssignRef, _TEXT
     mov     rcx, [rsi]
     mov     [rdi], rcx
 
     ;; Check whether the writes were even into the heap. If not there's no card update required.
     cmp     rdi, [g_lowest_address]
-    jb      RhpByRefAssignRef_NotInHeap
+    jb      RhpByRefAssignRef_NoBarrierRequired
     cmp     rdi, [g_highest_address]
-    jae     RhpByRefAssignRef_NotInHeap
+    jae     RhpByRefAssignRef_NoBarrierRequired
 
     ;; Update the shadow copy of the heap with the same value just written to the same heap. (A no-op unless
     ;; we're in a debug build and write barrier checking has been enabled).
     UPDATE_GC_SHADOW BASENAME, rcx, rdi
 
+ifdef FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
+    mov     r11, [g_write_watch_table]
+    cmp     r11, 0
+    je      RhpByRefAssignRef_CheckCardTable
+
+    mov     r10, rdi
+    shr     r10, 0Ch ;; SoftwareWriteWatch::AddressToTableByteIndexShift
+    add     r10, r11
+    cmp     byte ptr [r10], 0
+    jne     RhpByRefAssignRef_CheckCardTable
+    mov     byte ptr [r10], 0FFh
+endif
+
+RhpByRefAssignRef_CheckCardTable:
+
     ;; If the reference is to an object that's not in an ephemeral generation we have no need to track it
     ;; (since the object won't be collected or moved by an ephemeral collection).
     cmp     rcx, [g_ephemeral_low]
-    jb      RhpByRefAssignRef_NotInHeap
+    jb      RhpByRefAssignRef_NoBarrierRequired
     cmp     rcx, [g_ephemeral_high]
-    jae     RhpByRefAssignRef_NotInHeap
+    jae     RhpByRefAssignRef_NoBarrierRequired
 
-    ;; move current rdi value into rcx and then increment the pointers
+    ;; move current rdi value into rcx, we need to keep rdi and eventually increment by 8
     mov     rcx, rdi
-    add     rsi, 8h
-    add     rdi, 8h
 
     ;; We have a location on the GC heap being updated with a reference to an ephemeral object so we must
     ;; track this write. The location address is translated into an offset in the card table bitmap. We set
     ;; an entire byte in the card table since it's quicker than messing around with bitmasks and we only write
     ;; the byte if it hasn't already been done since writes are expensive and impact scaling.
-    shr     rcx, 11
-    add     rcx, [g_card_table]
-    cmp     byte ptr [rcx], 0FFh
-    jne     RhpByRefAssignRef_UpdateCardTable
-    ret
+    shr     rcx, 0Bh
+    mov     r10, [g_card_table]
+    cmp     byte ptr [rcx + r10], 0FFh
+    je      RhpByRefAssignRef_NoBarrierRequired
 
 ;; We get here if it's necessary to update the card table.
-RhpByRefAssignRef_UpdateCardTable:
+    mov     byte ptr [rcx + r10], 0FFh
+
+ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES
+    ;; Shift rcx by 0Ah more to get the card bundle byte (we shifted by 0Bh already)
+    shr     rcx, 0Ah
+    add     rcx, [g_card_bundle_table]
+    cmp     byte ptr [rcx], 0FFh
+    je      RhpByRefAssignRef_NoBarrierRequired
+
     mov     byte ptr [rcx], 0FFh
-    ret
+endif
 
-RhpByRefAssignRef_NotInHeap:
-    ; Increment the pointers before leaving
+RhpByRefAssignRef_NoBarrierRequired:
+    ;; Increment the pointers before leaving
     add     rdi, 8h
     add     rsi, 8h
     ret

src/coreclr/nativeaot/Runtime/arm64/AsmMacros.h

@@ -108,6 +108,14 @@ OFFSETOF__Thread__m_alloc_context__alloc_limit      equ OFFSETOF__Thread__m_rgbA
     EXTERN g_ephemeral_high
     EXTERN g_card_table
 
+#ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES
+    EXTERN g_card_bundle_table
+#endif
+
+#ifdef FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
+    EXTERN g_write_watch_table
+#endif
+
 
 ;; -----------------------------------------------------------------------------
 ;; Macro used to assign an alternate name to a symbol containing characters normally disallowed in a symbol
@@ -146,6 +154,15 @@ MovInstr SETS "movk"
         $MovInstr $Reg, #(($ConstantLo):AND:0xffff)
     MEND
 
+;;-----------------------------------------------------------------------------
+;; Macro for loading a 64bit value of a global variable into a register
+    MACRO
+        PREPARE_EXTERNAL_VAR_INDIRECT $Name, $Reg
+
+        adrp $Reg, $Name
+        ldr  $Reg, [$Reg, $Name]
+    MEND
+
 ;; -----------------------------------------------------------------------------
 ;;
 ;; Macro to export a pointer to an address inside a stub as a 64-bit variable