[GR-48504] [GR-48899] [GR-47055] Clean up virtual threads code.

substratevm/mx.substratevm/suite.py

@@ -1357,6 +1357,7 @@
             "requiresConcealed": {
                 "jdk.internal.vm.ci": [
                     "jdk.vm.ci.meta",
+                    "jdk.vm.ci.code",
                     "jdk.vm.ci.common",
                 ]
             },

substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ThreadLocalAllocation.java

@@ -67,7 +67,7 @@
 import com.oracle.svm.core.snippets.KnownIntrinsics;
 import com.oracle.svm.core.snippets.SubstrateForeignCallTarget;
 import com.oracle.svm.core.stack.StackOverflowCheck;
-import com.oracle.svm.core.thread.Continuation;
+import com.oracle.svm.core.thread.ContinuationSupport;
 import com.oracle.svm.core.thread.VMOperation;
 import com.oracle.svm.core.thread.VMThreads;
 import com.oracle.svm.core.threadlocal.FastThreadLocal;
@@ -375,7 +375,7 @@ private static Object allocateLargeArrayLikeObjectInNewTlab(DynamicHub hub, int
     @Uninterruptible(reason = "Holds uninitialized memory")
     private static Object formatArrayLikeObject(Pointer memory, DynamicHub hub, int length, boolean unaligned, FillContent fillContent, byte[] podReferenceMap) {
         Class<?> clazz = DynamicHub.toClass(hub);
-        if (Continuation.isSupported() && clazz == StoredContinuation.class) {
+        if (ContinuationSupport.isSupported() && clazz == StoredContinuation.class) {
             return FormatStoredContinuationNode.formatStoredContinuation(memory, clazz, length, false, unaligned, true);
         } else if (Pod.RuntimeSupport.isPresent() && podReferenceMap != null) {
             return FormatPodNode.formatPod(memory, clazz, length, podReferenceMap, false, unaligned, fillContent, true);

substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/graal/GenScavengeAllocationSnippets.java

@@ -58,7 +58,6 @@
 import com.oracle.svm.core.heap.Pod;
 import com.oracle.svm.core.hub.DynamicHub;
 import com.oracle.svm.core.hub.LayoutEncoding;
-import com.oracle.svm.core.thread.Continuation;
 import com.oracle.svm.core.thread.ContinuationSupport;
 
 import jdk.vm.ci.meta.JavaKind;
@@ -130,7 +129,7 @@ public Templates(OptionValues options, Providers providers, SubstrateAllocationS
             this.baseTemplates = baseTemplates;
             formatObject = snippet(providers, GenScavengeAllocationSnippets.class, "formatObjectSnippet");
             formatArray = snippet(providers, GenScavengeAllocationSnippets.class, "formatArraySnippet");
-            formatStoredContinuation = Continuation.isSupported() ? snippet(providers, GenScavengeAllocationSnippets.class, "formatStoredContinuation") : null;
+            formatStoredContinuation = ContinuationSupport.isSupported() ? snippet(providers, GenScavengeAllocationSnippets.class, "formatStoredContinuation") : null;
             formatPod = Pod.RuntimeSupport.isPresent() ? snippet(providers,
                             GenScavengeAllocationSnippets.class,
                             "formatPodSnippet",
@@ -141,7 +140,7 @@ public Templates(OptionValues options, Providers providers, SubstrateAllocationS
         public void registerLowering(Map<Class<? extends Node>, NodeLoweringProvider<?>> lowerings) {
             lowerings.put(FormatObjectNode.class, new FormatObjectLowering());
             lowerings.put(FormatArrayNode.class, new FormatArrayLowering());
-            if (Continuation.isSupported()) {
+            if (ContinuationSupport.isSupported()) {
                 lowerings.put(FormatStoredContinuationNode.class, new FormatStoredContinuationLowering());
             }
             if (Pod.RuntimeSupport.isPresent()) {

substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/graal/GenScavengeAllocationSupport.java

@@ -24,18 +24,19 @@
  */
 package com.oracle.svm.core.genscavenge.graal;
 
-import com.oracle.svm.core.heap.Pod;
-import com.oracle.svm.core.snippets.SnippetRuntime.SubstrateForeignCallDescriptor;
-import com.oracle.svm.core.thread.Continuation;
-import jdk.compiler.graal.core.common.spi.ForeignCallDescriptor;
-import jdk.compiler.graal.word.Word;
 import org.graalvm.word.UnsignedWord;
 
 import com.oracle.svm.core.genscavenge.HeapParameters;
 import com.oracle.svm.core.genscavenge.ThreadLocalAllocation;
 import com.oracle.svm.core.graal.meta.SubstrateForeignCallsProvider;
 import com.oracle.svm.core.graal.snippets.GCAllocationSupport;
+import com.oracle.svm.core.heap.Pod;
 import com.oracle.svm.core.snippets.SnippetRuntime;
+import com.oracle.svm.core.snippets.SnippetRuntime.SubstrateForeignCallDescriptor;
+import com.oracle.svm.core.thread.ContinuationSupport;
+
+import jdk.compiler.graal.core.common.spi.ForeignCallDescriptor;
+import jdk.compiler.graal.word.Word;
 
 public class GenScavengeAllocationSupport implements GCAllocationSupport {
     private static final SubstrateForeignCallDescriptor SLOW_NEW_INSTANCE = SnippetRuntime.findForeignCall(ThreadLocalAllocation.class, "slowPathNewInstance", true);
@@ -46,7 +47,7 @@ public class GenScavengeAllocationSupport implements GCAllocationSupport {
 
     public static void registerForeignCalls(SubstrateForeignCallsProvider foreignCalls) {
         foreignCalls.register(UNCONDITIONAL_FOREIGN_CALLS);
-        if (Continuation.isSupported()) {
+        if (ContinuationSupport.isSupported()) {
             foreignCalls.register(SLOW_NEW_STORED_CONTINUATION);
         }
         if (Pod.RuntimeSupport.isPresent()) {

substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/classinitialization/ClassInitializationInfo.java

@@ -29,7 +29,6 @@
 
 import org.graalvm.nativeimage.CurrentIsolate;
 import org.graalvm.nativeimage.IsolateThread;
-import com.oracle.svm.core.hub.PredefinedClassesSupport;
 import org.graalvm.nativeimage.Platform;
 import org.graalvm.nativeimage.Platforms;
 import org.graalvm.nativeimage.c.function.CFunctionPointer;
@@ -38,10 +37,12 @@
 import com.oracle.svm.core.FunctionPointerHolder;
 import com.oracle.svm.core.c.InvokeJavaFunctionPointer;
 import com.oracle.svm.core.hub.DynamicHub;
+import com.oracle.svm.core.hub.PredefinedClassesSupport;
 import com.oracle.svm.core.jdk.InternalVMMethod;
 import com.oracle.svm.core.snippets.SubstrateForeignCallTarget;
+import com.oracle.svm.core.thread.ContinuationSupport;
 import com.oracle.svm.core.thread.JavaThreads;
-import com.oracle.svm.core.thread.VirtualThreads;
+import com.oracle.svm.core.thread.Target_jdk_internal_vm_Continuation;
 import com.oracle.svm.core.util.VMError;
 
 import jdk.internal.misc.Unsafe;
@@ -263,16 +264,16 @@ private static void initialize(ClassInitializationInfo info, DynamicHub hub) {
         }
 
         boolean pinned = false;
-        if (VirtualThreads.isSupported() && JavaThreads.isCurrentThreadVirtual()) {
+        if (ContinuationSupport.isSupported() && JavaThreads.isCurrentThreadVirtual()) {
             // See comment on field `initThread`
-            VirtualThreads.singleton().pinCurrent();
+            Target_jdk_internal_vm_Continuation.pin();
             pinned = true;
         }
         try {
             doInitialize(info, hub);
         } finally {
             if (pinned) {
-                VirtualThreads.singleton().unpinCurrent();
+                Target_jdk_internal_vm_Continuation.unpin();
             }
         }
     }

substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/graal/snippets/SubstrateAllocationSnippets.java

@@ -102,7 +102,6 @@
 import com.oracle.svm.core.snippets.SnippetRuntime;
 import com.oracle.svm.core.snippets.SnippetRuntime.SubstrateForeignCallDescriptor;
 import com.oracle.svm.core.snippets.SubstrateForeignCallTarget;
-import com.oracle.svm.core.thread.Continuation;
 import com.oracle.svm.core.thread.ContinuationSupport;
 import com.oracle.svm.core.util.VMError;
 
@@ -600,7 +599,7 @@ public Templates(OptionValues options, Providers providers, SubstrateAllocationS
                             ALLOCATION_LOCATIONS);
 
             SnippetInfo allocateStoredContinuationSnippet = null;
-            if (Continuation.isSupported()) {
+            if (ContinuationSupport.isSupported()) {
                 allocateStoredContinuationSnippet = snippet(providers,
                                 SubstrateAllocationSnippets.class,
                                 "allocateStoredContinuation",
@@ -634,7 +633,7 @@ public void registerLowering(Map<Class<? extends Node>, NodeLoweringProvider<?>>
             lowerings.put(NewMultiArrayNode.class, new NewMultiArrayLowering());
             lowerings.put(ValidateNewInstanceClassNode.class, new ValidateNewInstanceClassLowering());
 
-            if (Continuation.isSupported()) {
+            if (ContinuationSupport.isSupported()) {
                 lowerings.put(NewStoredContinuationNode.class, new NewStoredContinuationLowering());
             }
             if (Pod.RuntimeSupport.isPresent()) {

substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/graal/stackvalue/StackValueNode.java

@@ -46,7 +46,6 @@
 import com.oracle.svm.core.FrameAccess;
 import com.oracle.svm.core.Uninterruptible;
 import com.oracle.svm.core.config.ConfigurationValues;
-import com.oracle.svm.core.thread.VirtualThreads;
 
 import jdk.vm.ci.meta.ResolvedJavaMethod;
 
@@ -153,7 +152,7 @@ public static StackValueNode create(int sizeInBytes, ResolvedJavaMethod method,
          * around in a caller, but these are difficult to ensure across multiple callers and
          * callees.
          */
-        boolean checkVirtualThread = disallowVirtualThread && VirtualThreads.isSupported() && !Uninterruptible.Utils.isUninterruptible(method);
+        boolean checkVirtualThread = disallowVirtualThread && !Uninterruptible.Utils.isUninterruptible(method);
         return create(sizeInBytes, slotIdentity, checkVirtualThread);
     }
 

substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/StoredContinuationAccess.java

@@ -30,7 +30,6 @@
 import jdk.compiler.graal.nodes.java.ArrayLengthNode;
 import jdk.compiler.graal.word.Word;
 import org.graalvm.nativeimage.ImageSingletons;
-import org.graalvm.nativeimage.IsolateThread;
 import org.graalvm.nativeimage.StackValue;
 import org.graalvm.nativeimage.c.function.CodePointer;
 import org.graalvm.nativeimage.c.struct.RawStructure;
@@ -39,9 +38,9 @@
 import org.graalvm.word.UnsignedWord;
 import org.graalvm.word.WordFactory;
 
-import com.oracle.svm.core.UnmanagedMemoryUtil;
 import com.oracle.svm.core.AlwaysInline;
 import com.oracle.svm.core.Uninterruptible;
+import com.oracle.svm.core.UnmanagedMemoryUtil;
 import com.oracle.svm.core.c.NonmovableArray;
 import com.oracle.svm.core.code.CodeInfo;
 import com.oracle.svm.core.code.CodeInfoAccess;
@@ -58,9 +57,10 @@
 import com.oracle.svm.core.stack.JavaStackWalk;
 import com.oracle.svm.core.stack.JavaStackWalker;
 import com.oracle.svm.core.stack.StackFrameVisitor;
-import com.oracle.svm.core.thread.Continuation;
+import com.oracle.svm.core.thread.ContinuationInternals;
 import com.oracle.svm.core.thread.ContinuationSupport;
 import com.oracle.svm.core.thread.Safepoint;
+import com.oracle.svm.core.thread.Target_jdk_internal_vm_Continuation;
 import com.oracle.svm.core.util.UnsignedUtils;
 import com.oracle.svm.core.util.VMError;
 
@@ -107,39 +107,14 @@ public static CodePointer getIP(StoredContinuation s) {
         return s.ip;
     }
 
-    public static int allocateToYield(Continuation c, Pointer baseSp, Pointer sp, CodePointer ip) {
-        assert sp.isNonNull() && ip.isNonNull();
-        return allocateFromStack(c, baseSp, sp, ip, WordFactory.nullPointer());
-    }
-
-    public static int allocateToPreempt(Continuation c, Pointer baseSp, IsolateThread targetThread) {
-        return allocateFromStack(c, baseSp, WordFactory.nullPointer(), WordFactory.nullPointer(), targetThread);
-    }
-
-    private static int allocateFromStack(Continuation cont, Pointer baseSp, Pointer sp, CodePointer ip, IsolateThread targetThread) {
-        boolean yield = sp.isNonNull();
-        assert yield == ip.isNonNull() && yield == targetThread.isNull();
-        assert baseSp.isNonNull();
-
-        Pointer startSp = sp;
-        CodePointer startIp = ip;
-        if (!yield) {
-            PreemptVisitor visitor = new PreemptVisitor(baseSp);
-            JavaStackWalker.walkThread(targetThread, visitor);
-            if (visitor.preemptStatus != Continuation.FREEZE_OK) {
-                return visitor.preemptStatus;
-            }
-            startSp = visitor.leafSP;
-            startIp = visitor.leafIP;
-        }
-
-        VMError.guarantee(startSp.isNonNull());
+    public static int allocateToYield(Target_jdk_internal_vm_Continuation c, Pointer baseSp, Pointer sp, CodePointer ip) {
+        assert baseSp.isNonNull() && sp.isNonNull() && ip.isNonNull();
 
-        int framesSize = UnsignedUtils.safeToInt(baseSp.subtract(startSp));
+        int framesSize = UnsignedUtils.safeToInt(baseSp.subtract(sp));
         StoredContinuation instance = allocate(framesSize);
-        fillUninterruptibly(instance, startIp, startSp, framesSize);
-        cont.stored = instance;
-        return Continuation.FREEZE_OK;
+        fillUninterruptibly(instance, ip, sp, framesSize);
+        ContinuationInternals.setStoredContinuation(c, instance);
+        return ContinuationSupport.FREEZE_OK;
     }
 
     @Uninterruptible(reason = "Prevent modifications to the stack while initializing instance and copying frames.")
@@ -300,7 +275,7 @@ private static final class PreemptVisitor extends StackFrameVisitor {
 
         Pointer leafSP;
         CodePointer leafIP;
-        int preemptStatus = Continuation.FREEZE_OK;
+        int preemptStatus = ContinuationSupport.FREEZE_OK;
 
         PreemptVisitor(Pointer endSP) {
             this.endSP = endSP;
@@ -315,7 +290,7 @@ protected boolean visitFrame(Pointer sp, CodePointer ip, CodeInfo codeInfo, Deop
             FrameInfoQueryResult frameInfo = CodeInfoTable.lookupCodeInfoQueryResult(codeInfo, ip).getFrameInfo();
             if (frameInfo.getSourceClass().equals(StoredContinuationAccess.class) && frameInfo.getSourceMethodName().equals("allocateToYield")) {
                 // Continuation is already in the process of yielding, cancel preemption.
-                preemptStatus = Continuation.YIELDING;
+                preemptStatus = ContinuationSupport.FREEZE_YIELDING;
                 return false;
             }
 

substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/hub/InteriorObjRefWalker.java

@@ -43,7 +43,7 @@
 import com.oracle.svm.core.heap.PodReferenceMapDecoder;
 import com.oracle.svm.core.heap.ReferenceAccess;
 import com.oracle.svm.core.heap.StoredContinuationAccess;
-import com.oracle.svm.core.thread.Continuation;
+import com.oracle.svm.core.thread.ContinuationSupport;
 import com.oracle.svm.core.util.VMError;
 
 /**
@@ -130,7 +130,7 @@ private static boolean walkPod(Object obj, ObjectReferenceVisitor visitor, Dynam
     @AlwaysInline("Performance critical version")
     @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true)
     private static boolean walkStoredContinuation(Object obj, ObjectReferenceVisitor visitor) {
-        if (!Continuation.isSupported()) {
+        if (!ContinuationSupport.isSupported()) {
             throw VMError.shouldNotReachHere("Stored continuation objects cannot be in the heap if the continuation support is disabled.");
         }
         return StoredContinuationAccess.walkReferences(obj, visitor);

substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/image/DisallowedImageHeapObjects.java

@@ -37,7 +37,6 @@
 import java.util.concurrent.ThreadLocalRandom;
 
 import com.oracle.svm.core.SubstrateUtil;
-import com.oracle.svm.core.thread.VirtualThreads;
 import com.oracle.svm.core.util.VMError;
 import com.oracle.svm.util.ReflectionUtil;
 
@@ -52,20 +51,12 @@ public interface DisallowedObjectReporter {
         RuntimeException raise(String msg, Object obj, String initializerAction);
     }
 
-    private static final Class<?> CANCELLABLE_CLASS;
-    private static final Class<?> JDK_VIRTUAL_THREAD_CLASS;
-    private static final Class<?> CONTINUATION_CLASS;
-    private static final Method CONTINUATION_IS_STARTED_METHOD;
-    private static final Class<?> CLEANER_CLEANABLE_CLASS;
-    private static final Class<?> LEGACY_CLEANER_CLASS;
-    static {
-        CANCELLABLE_CLASS = ReflectionUtil.lookupClass(false, "sun.nio.fs.Cancellable");
-        JDK_VIRTUAL_THREAD_CLASS = ReflectionUtil.lookupClass(true, "java.lang.VirtualThread");
-        CONTINUATION_CLASS = ReflectionUtil.lookupClass(true, "jdk.internal.vm.Continuation");
-        CONTINUATION_IS_STARTED_METHOD = (CONTINUATION_CLASS == null) ? null : ReflectionUtil.lookupMethod(CONTINUATION_CLASS, "isStarted");
-        CLEANER_CLEANABLE_CLASS = ReflectionUtil.lookupClass(false, "jdk.internal.ref.CleanerImpl$CleanerCleanable");
-        LEGACY_CLEANER_CLASS = ReflectionUtil.lookupClass(false, "jdk.internal.ref.Cleaner");
-    }
+    private static final Class<?> CANCELLABLE_CLASS = ReflectionUtil.lookupClass(false, "sun.nio.fs.Cancellable");
+    private static final Class<?> VIRTUAL_THREAD_CLASS = ReflectionUtil.lookupClass(false, "java.lang.VirtualThread");
+    private static final Class<?> CONTINUATION_CLASS = ReflectionUtil.lookupClass(false, "jdk.internal.vm.Continuation");
+    private static final Method CONTINUATION_IS_STARTED_METHOD = ReflectionUtil.lookupMethod(CONTINUATION_CLASS, "isStarted");
+    private static final Class<?> CLEANER_CLEANABLE_CLASS = ReflectionUtil.lookupClass(false, "jdk.internal.ref.CleanerImpl$CleanerCleanable");
+    private static final Class<?> LEGACY_CLEANER_CLASS = ReflectionUtil.lookupClass(false, "jdk.internal.ref.Cleaner");
 
     public static void check(Object obj, DisallowedObjectReporter reporter) {
         if (((obj instanceof Random) && !(obj instanceof ThreadLocalRandom)) || obj instanceof SplittableRandom) {
@@ -75,9 +66,8 @@ public static void check(Object obj, DisallowedObjectReporter reporter) {
         }
 
         /* Started platform threads */
-        if (obj instanceof Thread) {
-            final Thread asThread = (Thread) obj;
-            if (VirtualThreads.isSupported() && (VirtualThreads.singleton().isVirtual(asThread) || (JDK_VIRTUAL_THREAD_CLASS != null && JDK_VIRTUAL_THREAD_CLASS.isInstance(asThread)))) {
+        if (obj instanceof Thread asThread) {
+            if (VIRTUAL_THREAD_CLASS.isInstance(asThread)) {
                 // allowed unless the thread is mounted, in which case it references its carrier
                 // thread and fails
             } else if (asThread.getState() != Thread.State.NEW && asThread.getState() != Thread.State.TERMINATED) {
@@ -86,7 +76,7 @@ public static void check(Object obj, DisallowedObjectReporter reporter) {
                                 asThread, "Prevent threads from starting during image generation, or a started thread from being included in the image.");
             }
         }
-        if (SubstrateUtil.HOSTED && CONTINUATION_CLASS != null && CONTINUATION_CLASS.isInstance(obj)) {
+        if (SubstrateUtil.HOSTED && CONTINUATION_CLASS.isInstance(obj)) {
             boolean isStarted;
             try {
                 isStarted = (Boolean) CONTINUATION_IS_STARTED_METHOD.invoke(obj);