Skip to content

[release/7.0] Fix write barriers in NativeAOT #74455

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 1 commit into from
Aug 24, 2022
Merged

[release/7.0] Fix write barriers in NativeAOT #74455

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 2 additions & 1 deletion src/coreclr/nativeaot/Runtime/GCMemoryHelpers.inl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -229,7 +229,7 @@ FORCEINLINE void InlinedBulkWriteBarrier(void* pMemStart, size_t cbMemSize)
// Compute the shadow heap address corresponding to the beginning of the range of heap addresses modified
// and in the process range check it to make sure we have the shadow version allocated.
uintptr_t* shadowSlot = (uintptr_t*)(g_GCShadow + ((uint8_t*)pMemStart - g_lowest_address));
if (shadowSlot <= (uintptr_t*)g_GCShadowEnd)
if (shadowSlot < (uintptr_t*)g_GCShadowEnd)
{
// Iterate over every pointer sized slot in the range, copying data from the real heap to the shadow heap.
// As we perform each copy we need to recheck the real heap contents with an ordered read to ensure we're
Expand All @@ -239,6 +239,7 @@ FORCEINLINE void InlinedBulkWriteBarrier(void* pMemStart, size_t cbMemSize)

uintptr_t* realSlot = (uintptr_t*)pMemStart;
uintptr_t slotCount = cbMemSize / sizeof(uintptr_t);
ASSERT(slotCount < (uintptr_t*)g_GCShadowEnd - shadowSlot);
do
{
// Update shadow slot from real slot.
Expand Down
2 changes: 1 addition & 1 deletion src/coreclr/nativeaot/Runtime/amd64/WriteBarriers.S
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,7 @@
jb LOCAL_LABEL(\BASENAME\()_UpdateShadowHeap_PopThenDone_\REFREG)
add \DESTREG, [C_VAR(g_GCShadow)]
cmp \DESTREG, [C_VAR(g_GCShadowEnd)]
ja LOCAL_LABEL(\BASENAME\()_UpdateShadowHeap_PopThenDone_\REFREG)
jae LOCAL_LABEL(\BASENAME\()_UpdateShadowHeap_PopThenDone_\REFREG)

// Update the shadow heap.
mov [\DESTREG], \REFREG
Expand Down
2 changes: 1 addition & 1 deletion src/coreclr/nativeaot/Runtime/amd64/WriteBarriers.asm
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -43,7 +43,7 @@ UPDATE_GC_SHADOW macro BASENAME, REFREG, DESTREG
jb &BASENAME&_UpdateShadowHeap_PopThenDone_&REFREG&
add DESTREG, [g_GCShadow]
cmp DESTREG, [g_GCShadowEnd]
ja &BASENAME&_UpdateShadowHeap_PopThenDone_&REFREG&
jae &BASENAME&_UpdateShadowHeap_PopThenDone_&REFREG&

;; Update the shadow heap.
mov [DESTREG], REFREG
Expand Down
26 changes: 13 additions & 13 deletions src/coreclr/nativeaot/Runtime/arm/WriteBarriers.S
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -38,7 +38,7 @@
ldr r12, =C_FUNC(g_GCShadowEnd)
ldr r12, [r12]
cmp \DESTREG, r12
jhi LOCAL_LABEL(\BASENAME\()_UpdateShadowHeap_PopThenDone_\REFREG)
bhs LOCAL_LABEL(\BASENAME\()_UpdateShadowHeap_PopThenDone_\REFREG)

// Update the shadow heap.
str \REFREG, [\DESTREG]
Expand Down Expand Up @@ -105,15 +105,15 @@ LOCAL_LABEL(\BASENAME\()_UpdateShadowHeap_Done_\REFREG):

// If the reference is to an object that's not in an ephemeral generation we have no need to track it
// (since the object won't be collected or moved by an ephemeral collection).
ldr r12, =C_FUNC(g_ephemeral_low)
ldr r12, =C_FUNC(g_ephemeral_low)
ldr r12, [r12]
cmp \REFREG, r12
blo LOCAL_LABEL(\BASENAME\()_EXIT_\REFREG)

ldr r12, =C_FUNC(g_ephemeral_high)
ldr r12, =C_FUNC(g_ephemeral_high)
ldr r12, [r12]
cmp \REFREG, r12
bhi LOCAL_LABEL(\BASENAME\()_EXIT_\REFREG)
cmp \REFREG, r12
bhs LOCAL_LABEL(\BASENAME\()_EXIT_\REFREG)

// We have a location on the GC heap being updated with a reference to an ephemeral object so we must
// track this write. The location address is translated into an offset in the card table bitmap. We set
Expand Down Expand Up @@ -167,11 +167,11 @@ ALTERNATE_ENTRY RhpAssignRef
//
// Note that none of this is relevant for single cpu machines. We may choose to implement a
// uniprocessor specific version of this barrier if uni-proc becomes a significant scenario again.
dmb
dmb

// Write the reference into the location. Note that we rely on the fact that no GC can occur between here
// and the card table update we may perform below.
ALTERNATE_ENTRY "RhpAssignRefAvLocation"\EXPORT_REG_NAME // WriteBarrierFunctionAvLocation
ALTERNATE_ENTRY "RhpAssignRefAvLocation"\EXPORT_REG_NAME // WriteBarrierFunctionAvLocation
.ifc \REFREG, r1
ALTERNATE_ENTRY RhpAssignRefAVLocation
.endif
Expand All @@ -198,14 +198,14 @@ DEFINE_UNCHECKED_WRITE_BARRIER r1, r1

// The location being updated might not even lie in the GC heap (a handle or stack location for instance),
// in which case no write barrier is required.
ldr r12, =C_FUNC(g_lowest_address)
ldr r12, =C_FUNC(g_lowest_address)
ldr r12, [r12]
cmp r0, r12
blo LOCAL_LABEL(\BASENAME\()_NoBarrierRequired_\REFREG)
ldr r12, =C_FUNC(g_highest_address)
ldr r12, =C_FUNC(g_highest_address)
ldr r12, [r12]
cmp r0, r12
bhi LOCAL_LABEL(\BASENAME\()_NoBarrierRequired_\REFREG)
bhs LOCAL_LABEL(\BASENAME\()_NoBarrierRequired_\REFREG)

DEFINE_UNCHECKED_WRITE_BARRIER_CORE \BASENAME, \REFREG

Expand Down Expand Up @@ -270,7 +270,7 @@ LEAF_ENTRY RhpCheckedLockCmpXchg, _TEXT
// barrier must occur before the object reference update, so we have to do it unconditionally even
// though the update may fail below.
dmb
ALTERNATE_ENTRY RhpCheckedLockCmpXchgAVLocation
ALTERNATE_ENTRY RhpCheckedLockCmpXchgAVLocation
LOCAL_LABEL(RhpCheckedLockCmpXchgRetry):
ldrex r3, [r0]
cmp r2, r3
Expand Down Expand Up @@ -337,7 +337,7 @@ LEAF_ENTRY RhpByRefAssignRef, _TEXT
ldr r3, =C_FUNC(g_highest_address)
ldr r3, [r3]
cmp r0, r3
bhi LOCAL_LABEL(RhpByRefAssignRef_NotInHeap)
bhs LOCAL_LABEL(RhpByRefAssignRef_NotInHeap)

// Update the shadow copy of the heap with the same value just written to the same heap. (A no-op unless
// we're in a debug build and write barrier checking has been enabled).
Expand All @@ -352,7 +352,7 @@ LEAF_ENTRY RhpByRefAssignRef, _TEXT
ldr r3, =C_FUNC(g_ephemeral_high)
ldr r3, [r3]
cmp r2, r3
bhi LOCAL_LABEL(RhpByRefAssignRef_NotInHeap)
bhs LOCAL_LABEL(RhpByRefAssignRef_NotInHeap)

// move current r0 value into r2 and then increment the pointers
mov r2, r0
Expand Down
16 changes: 9 additions & 7 deletions src/coreclr/nativeaot/Runtime/arm64/WriteBarriers.S
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -38,14 +38,14 @@
// Transform destReg into the equivalent address in the shadow heap.
PREPARE_EXTERNAL_VAR_INDIRECT g_lowest_address, X9
subs \destReg, \destReg, x9
blt 0f
blo 0f

PREPARE_EXTERNAL_VAR_INDIRECT g_GCShadow, X9
add \destReg, \destReg, x9

PREPARE_EXTERNAL_VAR_INDIRECT g_GCShadowEnd, X9
cmp \destReg, x9
bgt 0f
bhs 0f

// Update the shadow heap.
str \refReg, [\destReg]
Expand Down Expand Up @@ -120,11 +120,11 @@
// an object not on the epehemeral segment.
PREPARE_EXTERNAL_VAR_INDIRECT g_ephemeral_low, x\trash
cmp \refReg, x\trash
blt 0f
blo 0f

PREPARE_EXTERNAL_VAR_INDIRECT g_ephemeral_high, x\trash
cmp \refReg, x\trash
bge 0f
bhs 0f

// Set this objects card, if it has not already been set.

Expand Down Expand Up @@ -172,11 +172,13 @@

PREPARE_EXTERNAL_VAR_INDIRECT g_lowest_address, x\trash
cmp \destReg, x\trash
blt 0f

PREPARE_EXTERNAL_VAR_INDIRECT g_highest_address, x\trash
cmp \destReg, x\trash
bgt 0f

// If \destReg >= g_lowest_address, compare \destReg to g_highest_address.
// Otherwise, set the C flag (0x2) to take the next branch.
ccmp \destReg, x\trash, #0x2, hs
bhs 0f

INSERT_UNCHECKED_WRITE_BARRIER_CORE \destReg, \refReg, \trash, \trash2

Expand Down
16 changes: 9 additions & 7 deletions src/coreclr/nativeaot/Runtime/arm64/WriteBarriers.asm
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -54,7 +54,7 @@ INVALIDGCVALUE EQU 0xCCCCCCCD
adrp x12, g_lowest_address
ldr x12, [x12, g_lowest_address]
subs $destReg, $destReg, x12
blt %ft0
blo %ft0

adrp x12, $g_GCShadow
ldr x12, [x12, $g_GCShadow]
Expand All @@ -63,7 +63,7 @@ INVALIDGCVALUE EQU 0xCCCCCCCD
adrp x12, $g_GCShadowEnd
ldr x12, [x12, $g_GCShadowEnd]
cmp $destReg, x12
bgt %ft0
bhs %ft0

;; Update the shadow heap.
str $refReg, [$destReg]
Expand Down Expand Up @@ -127,12 +127,12 @@ INVALIDGCVALUE EQU 0xCCCCCCCD
adrp x12, g_ephemeral_low
ldr x12, [x12, g_ephemeral_low]
cmp $refReg, x12
blt %ft0
blo %ft0

adrp x12, g_ephemeral_high
ldr x12, [x12, g_ephemeral_high]
cmp $refReg, x12
bge %ft0
bhs %ft0

;; Set this object's card, if it hasn't already been set.
adrp x12, g_card_table
Expand Down Expand Up @@ -170,12 +170,14 @@ INVALIDGCVALUE EQU 0xCCCCCCCD
adrp x12, g_lowest_address
ldr x12, [x12, g_lowest_address]
cmp $destReg, x12
blt %ft0

adrp x12, g_highest_address
ldr x12, [x12, g_highest_address]
cmp $destReg, x12
bgt %ft0

;; If $destReg >= g_lowest_address, compare $destReg to g_highest_address.
;; Otherwise, set the C flag (0x2) to take the next branch.
ccmp $destReg, x12, #0x2, hs
bhs %ft0

INSERT_UNCHECKED_WRITE_BARRIER_CORE $destReg, $refReg, $trashReg

Expand Down
2 changes: 1 addition & 1 deletion src/coreclr/nativeaot/Runtime/i386/WriteBarriers.asm
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -48,7 +48,7 @@ UPDATE_GC_SHADOW macro BASENAME, DESTREG, REFREG
jb &BASENAME&_UpdateShadowHeap_PopThenDone_&DESTREG&_&REFREG&
add DESTREG, [g_GCShadow]
cmp DESTREG, [g_GCShadowEnd]
ja &BASENAME&_UpdateShadowHeap_PopThenDone_&DESTREG&_&REFREG&
jae &BASENAME&_UpdateShadowHeap_PopThenDone_&DESTREG&_&REFREG&

;; Update the shadow heap.
mov [DESTREG], REFREG
Expand Down