[GR-45455] [GR-51055] Change how stack walking works.

substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/CompactingOldGeneration.java

@@ -25,12 +25,10 @@
 package com.oracle.svm.core.genscavenge;
 
 import static com.oracle.svm.core.snippets.KnownIntrinsics.readCallerStackPointer;
-import static com.oracle.svm.core.snippets.KnownIntrinsics.readReturnAddress;
 
 import org.graalvm.nativeimage.IsolateThread;
 import org.graalvm.nativeimage.Platform;
 import org.graalvm.nativeimage.Platforms;
-import org.graalvm.nativeimage.c.function.CodePointer;
 import org.graalvm.word.Pointer;
 import org.graalvm.word.UnsignedWord;
 
@@ -365,8 +363,7 @@ private void fixupUnalignedChunkReferences(ChunkReleaser chunkReleaser) {
     @Uninterruptible(reason = "Required by called JavaStackWalker methods. We are at a safepoint during GC, so it does not change anything for this method.")
     private void fixupStackReferences() {
         Pointer sp = readCallerStackPointer();
-        CodePointer ip = readReturnAddress();
-        GCImpl.walkStackRoots(refFixupVisitor, sp, ip, false);
+        GCImpl.walkStackRoots(refFixupVisitor, sp, false);
     }
 
     private void compact(Timers timers) {

substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java

@@ -25,8 +25,6 @@
 package com.oracle.svm.core.genscavenge;
 
 import static com.oracle.svm.core.Uninterruptible.CALLED_FROM_UNINTERRUPTIBLE_CODE;
-import static com.oracle.svm.core.snippets.KnownIntrinsics.readCallerStackPointer;
-import static com.oracle.svm.core.snippets.KnownIntrinsics.readReturnAddress;
 
 import java.lang.ref.Reference;
 
@@ -35,7 +33,6 @@
 import org.graalvm.nativeimage.Platform;
 import org.graalvm.nativeimage.Platforms;
 import org.graalvm.nativeimage.StackValue;
-import org.graalvm.nativeimage.c.function.CodePointer;
 import org.graalvm.nativeimage.c.struct.RawField;
 import org.graalvm.nativeimage.c.struct.RawStructure;
 import org.graalvm.nativeimage.c.struct.SizeOf;
@@ -57,7 +54,6 @@
 import com.oracle.svm.core.code.CodeInfoTable;
 import com.oracle.svm.core.code.RuntimeCodeInfoAccess;
 import com.oracle.svm.core.code.RuntimeCodeInfoMemory;
-import com.oracle.svm.core.code.SimpleCodeInfoQueryResult;
 import com.oracle.svm.core.deopt.DeoptimizedFrame;
 import com.oracle.svm.core.deopt.Deoptimizer;
 import com.oracle.svm.core.genscavenge.AlignedHeapChunk.AlignedHeader;
@@ -88,6 +84,9 @@
 import com.oracle.svm.core.log.Log;
 import com.oracle.svm.core.os.CommittedMemoryProvider;
 import com.oracle.svm.core.snippets.ImplicitExceptions;
+import com.oracle.svm.core.snippets.KnownIntrinsics;
+import com.oracle.svm.core.stack.JavaFrame;
+import com.oracle.svm.core.stack.JavaFrames;
 import com.oracle.svm.core.stack.JavaStackWalk;
 import com.oracle.svm.core.stack.JavaStackWalker;
 import com.oracle.svm.core.stack.StackOverflowCheck;
@@ -801,95 +800,72 @@ private static PinnedObjectImpl removeClosedPinnedObjects(PinnedObjectImpl list)
     private void blackenStackRoots() {
         Timer blackenStackRootsTimer = timers.blackenStackRoots.open();
         try {
-            Pointer sp = readCallerStackPointer();
-            CodePointer ip = readReturnAddress();
-
-            walkStackRoots(greyToBlackObjRefVisitor, sp, ip, true);
+            Pointer sp = KnownIntrinsics.readCallerStackPointer();
+            walkStackRoots(greyToBlackObjRefVisitor, sp, true);
         } finally {
             blackenStackRootsTimer.close();
         }
     }
 
     @AlwaysInline("GC performance")
     @Uninterruptible(reason = "Required by called JavaStackWalker methods. We are at a safepoint during GC, so it does not change anything for this method.", mayBeInlined = true)
-    static void walkStackRoots(ObjectReferenceVisitor visitor, Pointer currentThreadSp, CodePointer currentThreadIp, boolean visitRuntimeCodeInfo) {
-        JavaStackWalk walk = StackValue.get(JavaStackWalk.class);
-        JavaStackWalker.initWalk(walk, currentThreadSp, currentThreadIp);
-        walkStack(walk, visitor, visitRuntimeCodeInfo);
+    static void walkStackRoots(ObjectReferenceVisitor visitor, Pointer currentThreadSp, boolean visitRuntimeCodeInfo) {
+        /*
+         * Walk the current thread (unlike all other threads, it does not have a usable frame
+         * anchor).
+         */
+        JavaStackWalk walk = StackValue.get(JavaStackWalker.sizeOfJavaStackWalk());
+        JavaStackWalker.initialize(walk, CurrentIsolate.getCurrentThread(), currentThreadSp);
+        walkStack(CurrentIsolate.getCurrentThread(), walk, visitor, visitRuntimeCodeInfo);
 
         /*
          * Scan the stacks of all the threads. Other threads will be blocked at a safepoint (or in
          * native code) so they will each have a JavaFrameAnchor in their VMThread.
          */
-        for (IsolateThread vmThread = VMThreads.firstThread(); vmThread.isNonNull(); vmThread = VMThreads.nextThread(vmThread)) {
-            if (vmThread == CurrentIsolate.getCurrentThread()) {
-                /*
-                 * The current thread is already scanned by code above, so we do not have to do
-                 * anything for it here. It might have a JavaFrameAnchor from earlier Java-to-C
-                 * transitions, but certainly not at the top of the stack since it is running this
-                 * code, so just this scan would be incomplete.
-                 */
+        for (IsolateThread thread = VMThreads.firstThread(); thread.isNonNull(); thread = VMThreads.nextThread(thread)) {
+            if (thread == CurrentIsolate.getCurrentThread()) {
                 continue;
             }
-            if (JavaStackWalker.initWalk(walk, vmThread)) {
-                walkStack(walk, visitor, visitRuntimeCodeInfo);
-            }
+            JavaStackWalker.initialize(walk, thread);
+            walkStack(thread, walk, visitor, visitRuntimeCodeInfo);
         }
     }
 
-    /**
-     * This method inlines {@link JavaStackWalker#continueWalk(JavaStackWalk, CodeInfo)} and
-     * {@link CodeInfoTable#visitObjectReferences}. This avoids looking up the
-     * {@link SimpleCodeInfoQueryResult} twice per frame, and also ensures that there are no virtual
-     * calls to a stack frame visitor.
-     */
     @AlwaysInline("GC performance")
     @Uninterruptible(reason = "Required by called JavaStackWalker methods. We are at a safepoint during GC, so it does not change anything for this method.", mayBeInlined = true)
-    private static void walkStack(JavaStackWalk walk, ObjectReferenceVisitor visitor, boolean visitRuntimeCodeInfo) {
+    private static void walkStack(IsolateThread thread, JavaStackWalk walk, ObjectReferenceVisitor visitor, boolean visitRuntimeCodeInfo) {
         assert VMOperation.isGCInProgress() : "This methods accesses a CodeInfo without a tether";
 
-        while (true) {
-            SimpleCodeInfoQueryResult queryResult = StackValue.get(SimpleCodeInfoQueryResult.class);
-            Pointer sp = walk.getSP();
-            CodePointer ip = walk.getPossiblyStaleIP();
+        while (JavaStackWalker.advance(walk, thread)) {
+            JavaFrame frame = JavaStackWalker.getCurrentFrame(walk);
+            VMError.guarantee(!JavaFrames.isUnknownFrame(frame), "GC must not encounter unknown frames");
 
             /* We are during a GC, so tethering of the CodeInfo is not necessary. */
-            CodeInfo codeInfo = CodeInfoAccess.convert(walk.getIPCodeInfo());
-            DeoptimizedFrame deoptFrame = Deoptimizer.checkDeoptimized(sp);
+            DeoptimizedFrame deoptFrame = Deoptimizer.checkDeoptimized(frame);
             if (deoptFrame == null) {
-                if (codeInfo.isNull()) {
-                    throw JavaStackWalker.reportUnknownFrameEncountered(sp, ip, deoptFrame);
-                }
-
-                CodeInfoAccess.lookupCodeInfo(codeInfo, CodeInfoAccess.relativeIP(codeInfo, ip), queryResult);
-                assert Deoptimizer.checkDeoptimized(sp) == null : "We are at a safepoint, so no deoptimization can have happened";
-
+                Pointer sp = frame.getSP();
+                CodeInfo codeInfo = CodeInfoAccess.unsafeConvert(frame.getIPCodeInfo());
                 NonmovableArray<Byte> referenceMapEncoding = CodeInfoAccess.getStackReferenceMapEncoding(codeInfo);
-                long referenceMapIndex = queryResult.getReferenceMapIndex();
+                long referenceMapIndex = frame.getReferenceMapIndex();
                 if (referenceMapIndex == ReferenceMapIndex.NO_REFERENCE_MAP) {
-                    throw CodeInfoTable.reportNoReferenceMap(sp, ip, codeInfo);
+                    throw CodeInfoTable.fatalErrorNoReferenceMap(sp, frame.getIP(), codeInfo);
                 }
                 CodeReferenceMapDecoder.walkOffsetsFromPointer(sp, referenceMapEncoding, referenceMapIndex, visitor, null);
+
+                if (RuntimeCompilation.isEnabled() && visitRuntimeCodeInfo && !CodeInfoAccess.isAOTImageCode(codeInfo)) {
+                    /*
+                     * Runtime-compiled code that is currently on the stack must be kept alive. So,
+                     * we mark the tether as strongly reachable. The RuntimeCodeCacheWalker will
+                     * handle all other object references later on.
+                     */
+                    RuntimeCodeInfoAccess.walkTether(codeInfo, visitor);
+                }
             } else {
                 /*
                  * This is a deoptimized frame. The DeoptimizedFrame object is stored in the frame,
                  * but it is pinned so we do not need to visit references of the frame.
                  */
             }
-
-            if (RuntimeCompilation.isEnabled() && visitRuntimeCodeInfo && !CodeInfoAccess.isAOTImageCode(codeInfo)) {
-                /*
-                 * Runtime-compiled code that is currently on the stack must be kept alive. So, we
-                 * mark the tether as strongly reachable. The RuntimeCodeCacheWalker will handle all
-                 * other object references later on.
-                 */
-                RuntimeCodeInfoAccess.walkTether(codeInfo, visitor);
-            }
-
-            if (!JavaStackWalker.continueWalk(walk, queryResult, deoptFrame)) {
-                /* No more caller frame found. */
-                return;
-            }
         }
     }
 
@@ -980,7 +956,7 @@ private void blackenImageHeapRoots() {
         }
     }
 
-    @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true)
+    @Uninterruptible(reason = "Forced inlining (StoredContinuation objects must not move).")
     private void blackenImageHeapRoots(ImageHeapInfo imageHeapInfo) {
         walkImageHeapRoots(imageHeapInfo, greyToBlackObjectVisitor);
     }

substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GreyToBlackObjectVisitor.java

@@ -56,7 +56,7 @@ public boolean visitObject(Object o) {
 
     @Override
     @AlwaysInline("GC performance")
-    @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true)
+    @Uninterruptible(reason = "Forced inlining (StoredContinuation objects must not move).", callerMustBe = true)
     public boolean visitObjectInline(Object o) {
         ReferenceObjectProcessing.discoverIfReference(o, objRefVisitor);
         InteriorObjRefWalker.walkObjectInline(o, objRefVisitor);

substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/HeapChunk.java

@@ -294,12 +294,13 @@ private static SignedWord offsetFromPointer(Header<?> that, PointerBase pointer)
     }
 
     @NeverInline("Not performance critical")
+    @Uninterruptible(reason = "Forced inlining (StoredContinuation objects must not move).")
     public static boolean walkObjectsFrom(Header<?> that, Pointer start, ObjectVisitor visitor) {
         return walkObjectsFromInline(that, start, visitor);
     }
 
     @AlwaysInline("GC performance")
-    @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true)
+    @Uninterruptible(reason = "Forced inlining (StoredContinuation objects must not move).", callerMustBe = true)
     public static boolean walkObjectsFromInline(Header<?> that, Pointer start, ObjectVisitor visitor) {
         Pointer p = start;
         while (p.belowThan(getTopPointer(that))) { // crucial: top can move, so always re-read
@@ -312,6 +313,7 @@ public static boolean walkObjectsFromInline(Header<?> that, Pointer start, Objec
         return true;
     }
 
+    @AlwaysInline("de-virtualize calls to ObjectReferenceVisitor")
     @Uninterruptible(reason = "Bridge between uninterruptible and potentially interruptible code.", mayBeInlined = true, calleeMustBe = false)
     private static boolean callVisitor(ObjectVisitor visitor, Object obj) {
         return visitor.visitObjectInline(obj);

substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ImageHeapWalker.java

@@ -66,18 +66,19 @@ public static boolean walkImageHeapObjects(ImageHeapInfo heapInfo, ObjectVisitor
                         walkPartition(heapInfo.firstReadOnlyHugeObject, heapInfo.lastReadOnlyHugeObject, visitor, false);
     }
 
+    @Uninterruptible(reason = "Forced inlining (StoredContinuation objects must not move).")
     static boolean walkPartition(Object firstObject, Object lastObject, ObjectVisitor visitor, boolean alignedChunks) {
         return walkPartitionInline(firstObject, lastObject, visitor, alignedChunks, false);
     }
 
     @AlwaysInline("GC performance")
-    @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true)
+    @Uninterruptible(reason = "Forced inlining (StoredContinuation objects must not move).", callerMustBe = true)
     static boolean walkPartitionInline(Object firstObject, Object lastObject, ObjectVisitor visitor, boolean alignedChunks) {
         return walkPartitionInline(firstObject, lastObject, visitor, alignedChunks, true);
     }
 
     @AlwaysInline("GC performance")
-    @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true)
+    @Uninterruptible(reason = "Forced inlining (StoredContinuation objects must not move).", callerMustBe = true)
     private static boolean walkPartitionInline(Object firstObject, Object lastObject, ObjectVisitor visitor, boolean alignedChunks, boolean inlineObjectVisit) {
         if (firstObject == null || lastObject == null) {
             assert firstObject == null && lastObject == null;
@@ -129,6 +130,7 @@ private static boolean visitObject(ObjectVisitor visitor, Object currentObject)
         return visitor.visitObject(currentObject);
     }
 
+    @AlwaysInline("de-virtualize calls to ObjectReferenceVisitor")
     @Uninterruptible(reason = "Bridge between uninterruptible and potentially interruptible code.", mayBeInlined = true, calleeMustBe = false)
     private static boolean visitObjectInline(ObjectVisitor visitor, Object currentObject) {
         return visitor.visitObjectInline(currentObject);
@@ -178,10 +180,9 @@ public boolean consistsOfHugeObjects(ImageHeapInfo region) {
     }
 
     @Override
-    @AlwaysInline("GC performance")
     public final boolean visitObjects(ImageHeapInfo region, ObjectVisitor visitor) {
         boolean alignedChunks = !consistsOfHugeObjects;
-        return ImageHeapWalker.walkPartitionInline(getFirstObject(region), getLastObject(region), visitor, alignedChunks);
+        return ImageHeapWalker.walkPartition(getFirstObject(region), getLastObject(region), visitor, alignedChunks);
     }
 
     protected abstract Object getFirstObject(ImageHeapInfo info);

substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/PathExhibitor.java

@@ -32,7 +32,6 @@
 import org.graalvm.nativeimage.c.function.CodePointer;
 import org.graalvm.word.Pointer;
 import org.graalvm.word.UnsignedWord;
-import org.graalvm.word.WordFactory;
 
 import com.oracle.svm.core.NeverInline;
 import com.oracle.svm.core.code.CodeInfo;
@@ -272,23 +271,28 @@ void reset() {
 
         @Override
         @RestrictHeapAccess(access = RestrictHeapAccess.Access.NO_ALLOCATION, reason = "Must not allocate while visiting stack frames.")
-        public boolean visitFrame(Pointer sp, CodePointer ip, CodeInfo codeInfo, DeoptimizedFrame deoptimizedFrame) {
-            frameSlotVisitor.initialize(ip, deoptimizedFrame, target, result);
-            return CodeInfoTable.visitObjectReferences(sp, ip, codeInfo, deoptimizedFrame, frameSlotVisitor);
+        public boolean visitRegularFrame(Pointer sp, CodePointer ip, CodeInfo codeInfo) {
+            frameSlotVisitor.initialize(ip, target, result);
+            return CodeInfoTable.visitObjectReferences(sp, ip, codeInfo, frameSlotVisitor);
+        }
+
+        @Override
+        @RestrictHeapAccess(access = RestrictHeapAccess.Access.NO_ALLOCATION, reason = "Must not allocate while visiting stack frames.")
+        protected boolean visitDeoptimizedFrame(Pointer originalSP, CodePointer deoptStubIP, DeoptimizedFrame deoptimizedFrame) {
+            /* Support for deoptimized frames is not implemented at the moment. */
+            return true;
         }
     }
 
     private static class FrameSlotVisitor extends AbstractVisitor implements ObjectReferenceVisitor {
         private CodePointer ip;
-        private DeoptimizedFrame deoptFrame;
 
         FrameSlotVisitor() {
         }
 
-        void initialize(CodePointer ipArg, DeoptimizedFrame deoptFrameArg, TargetMatcher targetMatcher, PathEdge edge) {
+        void initialize(CodePointer ipArg, TargetMatcher targetMatcher, PathEdge edge) {
             super.initialize(targetMatcher, edge);
             ip = ipArg;
-            deoptFrame = deoptFrameArg;
         }
 
         @Override
@@ -300,7 +304,7 @@ public boolean visitObjectReference(Pointer stackSlot, boolean compressed, Objec
             Pointer referentPointer = ReferenceAccess.singleton().readObjectAsUntrackedPointer(stackSlot, compressed);
             trace.string("  referentPointer: ").zhex(referentPointer);
             if (target.matches(referentPointer.toObject())) {
-                result.fill(new StackElement(stackSlot, ip, deoptFrame), new LeafElement(referentPointer.toObject()));
+                result.fill(new StackElement(stackSlot, ip), new LeafElement(referentPointer.toObject()));
                 return false;
             }
             return true;
@@ -434,12 +438,10 @@ public Log toLog(Log log) {
     public static class StackElement extends PathElement {
         private final Pointer stackSlot;
         private final CodePointer ip;
-        private final CodePointer deoptSourcePC;
         private final Pointer slotValue;
 
-        StackElement(Pointer stackSlot, CodePointer ip, DeoptimizedFrame deoptFrame) {
+        StackElement(Pointer stackSlot, CodePointer ip) {
             this.stackSlot = stackSlot;
-            this.deoptSourcePC = deoptFrame != null ? deoptFrame.getSourcePC() : WordFactory.nullPointer();
             this.ip = ip;
             this.slotValue = stackSlot.readWord(0);
         }
@@ -454,7 +456,6 @@ public Object getObject() {
         public Log toLog(Log log) {
             log.string("[stack:");
             log.string("  slot: ").zhex(stackSlot);
-            log.string("  deoptSourcePC: ").zhex(deoptSourcePC);
             log.string("  ip: ").zhex(ip);
             log.string("  value: ").zhex(slotValue);
             log.string("]");

substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/StackVerifier.java

@@ -86,12 +86,19 @@ public void initialize() {
 
         @Override
         @RestrictHeapAccess(access = RestrictHeapAccess.Access.NO_ALLOCATION, reason = "Must not allocate while verifying the stack.")
-        public boolean visitFrame(Pointer currentSP, CodePointer currentIP, CodeInfo codeInfo, DeoptimizedFrame deoptimizedFrame) {
+        public boolean visitRegularFrame(Pointer currentSP, CodePointer currentIP, CodeInfo codeInfo) {
             verifyFrameReferencesVisitor.initialize();
-            CodeInfoTable.visitObjectReferences(currentSP, currentIP, codeInfo, deoptimizedFrame, verifyFrameReferencesVisitor);
+            CodeInfoTable.visitObjectReferences(currentSP, currentIP, codeInfo, verifyFrameReferencesVisitor);
             result &= verifyFrameReferencesVisitor.result;
             return true;
         }
+
+        @Override
+        @RestrictHeapAccess(access = RestrictHeapAccess.Access.NO_ALLOCATION, reason = "Must not allocate while verifying the stack.")
+        protected boolean visitDeoptimizedFrame(Pointer originalSP, CodePointer deoptStubIP, DeoptimizedFrame deoptimizedFrame) {
+            /* Nothing to do. */
+            return true;
+        }
     }
 
     private static class VerifyFrameReferencesVisitor implements ObjectReferenceVisitor {