8293613: need to properly handle and hide tmp VTMS transitions

Reviewed-by: cjplummer, lmesnik

Author: Serguei Spitsyn

make/data/hotspot-symbols/symbols-unix

@@ -224,4 +224,5 @@ JVM_VirtualThreadMountBegin
 JVM_VirtualThreadMountEnd
 JVM_VirtualThreadUnmountBegin
 JVM_VirtualThreadUnmountEnd
+JVM_VirtualThreadHideFrames
 #

src/hotspot/share/include/jvm.h

@@ -1150,6 +1150,9 @@ JVM_VirtualThreadUnmountBegin(JNIEnv* env, jobject vthread, jboolean last_unmoun
 JNIEXPORT void JNICALL
 JVM_VirtualThreadUnmountEnd(JNIEnv* env, jobject vthread, jboolean last_unmount);
 
+JNIEXPORT void JNICALL
+JVM_VirtualThreadHideFrames(JNIEnv* env, jobject vthread, jboolean hide);
+
 /*
  * Core reflection support.
  */

src/hotspot/share/prims/jvm.cpp

@@ -3882,6 +3882,8 @@ JVM_ENTRY(void, JVM_VirtualThreadMountBegin(JNIEnv* env, jobject vthread, jboole
     assert(!JvmtiExport::can_support_virtual_threads(), "sanity check");
     return;
   }
+  assert(!thread->is_in_tmp_VTMS_transition(), "sanity check");
+  assert(!thread->is_in_VTMS_transition(), "sanity check");
   JvmtiVTMSTransitionDisabler::start_VTMS_transition(vthread, /* is_mount */ true);
 #else
   fatal("Should only be called with JVMTI enabled");
@@ -3906,6 +3908,7 @@ JVM_ENTRY(void, JVM_VirtualThreadMountEnd(JNIEnv* env, jobject vthread, jboolean
     }
   }
   assert(thread->is_in_VTMS_transition(), "sanity check");
+  assert(!thread->is_in_tmp_VTMS_transition(), "sanity check");
   JvmtiVTMSTransitionDisabler::finish_VTMS_transition(vthread, /* is_mount */ true);
   if (first_mount) {
     // thread start
@@ -3954,7 +3957,7 @@ JVM_ENTRY(void, JVM_VirtualThreadUnmountBegin(JNIEnv* env, jobject vthread, jboo
       }
     }
   }
-
+  assert(!thread->is_in_tmp_VTMS_transition(), "sanity check");
   assert(!thread->is_in_VTMS_transition(), "sanity check");
   JvmtiVTMSTransitionDisabler::start_VTMS_transition(vthread, /* is_mount */ false);
 
@@ -3977,12 +3980,27 @@ JVM_ENTRY(void, JVM_VirtualThreadUnmountEnd(JNIEnv* env, jobject vthread, jboole
     return;
   }
   assert(thread->is_in_VTMS_transition(), "sanity check");
+  assert(!thread->is_in_tmp_VTMS_transition(), "sanity check");
   JvmtiVTMSTransitionDisabler::finish_VTMS_transition(vthread, /* is_mount */ false);
 #else
   fatal("Should only be called with JVMTI enabled");
 #endif
 JVM_END
 
+JVM_ENTRY(void, JVM_VirtualThreadHideFrames(JNIEnv* env, jobject vthread, jboolean hide))
+#if INCLUDE_JVMTI
+  if (!DoJVMTIVirtualThreadTransitions) {
+    assert(!JvmtiExport::can_support_virtual_threads(), "sanity check");
+    return;
+  }
+  assert(!thread->is_in_VTMS_transition(), "sanity check");
+  assert(thread->is_in_tmp_VTMS_transition() != (bool)hide, "sanity check");
+  thread->toggle_is_in_tmp_VTMS_transition();
+#else
+  fatal("Should only be called with JVMTI enabled");
+#endif
+JVM_END
+
 /*
  * Return the current class's class file version.  The low order 16 bits of the
  * returned jint contain the class's major version.  The high order 16 bits

src/hotspot/share/prims/jvmtiEnvBase.cpp

@@ -703,10 +703,14 @@ JvmtiEnvBase::get_vthread_jvf(oop vthread) {
 javaVFrame*
 JvmtiEnvBase::get_cthread_last_java_vframe(JavaThread* jt, RegisterMap* reg_map_p) {
   // Strip vthread frames in case of carrier thread with mounted continuation.
-  javaVFrame *jvf = JvmtiEnvBase::is_cthread_with_continuation(jt) ?
-                        jt->carrier_last_java_vframe(reg_map_p) :
-                        jt->last_java_vframe(reg_map_p);
-  jvf = check_and_skip_hidden_frames(jt, jvf);
+  bool cthread_with_cont = JvmtiEnvBase::is_cthread_with_continuation(jt);
+  javaVFrame *jvf = cthread_with_cont ? jt->carrier_last_java_vframe(reg_map_p)
+                                      : jt->last_java_vframe(reg_map_p);
+  // Skip hidden frames only for carrier threads
+  // which are in non-temporary VTMS transition.
+  if (jt->is_in_VTMS_transition()) {
+    jvf = check_and_skip_hidden_frames(jt, jvf);
+  }
   return jvf;
 }
 

src/hotspot/share/prims/jvmtiExport.cpp

@@ -893,7 +893,7 @@ class JvmtiClassFileLoadHookPoster : public StackObj {
     _cached_class_file_ptr = cache_ptr;
     _has_been_modified = false;
 
-    assert(!_thread->is_in_VTMS_transition(), "CFLH events are not allowed in VTMS transition");
+    assert(!_thread->is_in_any_VTMS_transition(), "CFLH events are not allowed in any VTMS transition");
     _state = _thread->jvmti_thread_state();
     if (_state != NULL) {
       _class_being_redefined = _state->get_class_being_redefined();
@@ -1050,6 +1050,9 @@ bool JvmtiExport::post_class_file_load_hook(Symbol* h_name,
   if (JvmtiEnv::get_phase() < JVMTI_PHASE_PRIMORDIAL) {
     return false;
   }
+  if (JavaThread::current()->is_in_tmp_VTMS_transition()) {
+    return false; // skip CFLH events in tmp VTMS transition
+  }
 
   JvmtiClassFileLoadHookPoster poster(h_name, class_loader,
                                       h_protection_domain,
@@ -1184,8 +1187,8 @@ void JvmtiExport::post_raw_breakpoint(JavaThread *thread, Method* method, addres
   if (state == NULL) {
     return;
   }
-  if (thread->is_in_VTMS_transition()) {
-    return; // no events should be posted if thread is in a VTMS transition
+  if (thread->is_in_any_VTMS_transition()) {
+    return; // no events should be posted if thread is in any VTMS transition
   }
 
   EVT_TRIG_TRACE(JVMTI_EVENT_BREAKPOINT, ("[%s] Trg Breakpoint triggered",
@@ -1340,7 +1343,10 @@ void JvmtiExport::post_class_load(JavaThread *thread, Klass* klass) {
   if (state == NULL) {
     return;
   }
-  assert(!thread->is_in_VTMS_transition(), "class load events are not allowed in VTMS transition");
+  if (thread->is_in_tmp_VTMS_transition()) {
+    return; // skip ClassLoad events in tmp VTMS transition
+  }
+  assert(!thread->is_in_any_VTMS_transition(), "class load events are not allowed in any VTMS transition");
 
   EVT_TRIG_TRACE(JVMTI_EVENT_CLASS_LOAD, ("[%s] Trg Class Load triggered",
                       JvmtiTrace::safe_get_thread_name(thread)));
@@ -1375,7 +1381,10 @@ void JvmtiExport::post_class_prepare(JavaThread *thread, Klass* klass) {
   if (state == NULL) {
     return;
   }
-  assert(!thread->is_in_VTMS_transition(), "class prepare events are not allowed in VTMS transition");
+  if (thread->is_in_tmp_VTMS_transition()) {
+    return; // skip ClassPrepare events in tmp VTMS transition
+  }
+  assert(!thread->is_in_any_VTMS_transition(), "class prepare events are not allowed in any VTMS transition");
 
   EVT_TRIG_TRACE(JVMTI_EVENT_CLASS_PREPARE, ("[%s] Trg Class Prepare triggered",
                       JvmtiTrace::safe_get_thread_name(thread)));
@@ -1694,8 +1703,8 @@ void JvmtiExport::post_object_free(JvmtiEnv* env, GrowableArray<jlong>* objects)
   assert(objects != NULL, "Nothing to post");
 
   JavaThread *javaThread = JavaThread::current();
-  if (javaThread->is_in_VTMS_transition()) {
-    return; // no events should be posted if thread is in a VTMS transition
+  if (javaThread->is_in_any_VTMS_transition()) {
+    return; // no events should be posted if thread is in any VTMS transition
   }
   if (!env->is_enabled(JVMTI_EVENT_OBJECT_FREE)) {
     return; // the event type has been already disabled
@@ -1718,8 +1727,8 @@ void JvmtiExport::post_resource_exhausted(jint resource_exhausted_flags, const c
 
   JavaThread *thread  = JavaThread::current();
 
-  if (thread->is_in_VTMS_transition()) {
-    return; // no events should be posted if thread is in a VTMS transition
+  if (thread->is_in_any_VTMS_transition()) {
+    return; // no events should be posted if thread is in any VTMS transition
   }
 
   log_error(jvmti)("Posting Resource Exhausted event: %s",
@@ -1761,8 +1770,8 @@ void JvmtiExport::post_method_entry(JavaThread *thread, Method* method, frame cu
     // for any thread that actually wants method entry, interp_only_mode is set
     return;
   }
-  if (mh->jvmti_mount_transition() || thread->is_in_VTMS_transition()) {
-    return; // no events should be posted if thread is in a VTMS transition
+  if (mh->jvmti_mount_transition() || thread->is_in_any_VTMS_transition()) {
+    return; // no events should be posted if thread is in any VTMS transition
   }
   EVT_TRIG_TRACE(JVMTI_EVENT_METHOD_ENTRY, ("[%s] Trg Method Entry triggered %s.%s",
                      JvmtiTrace::safe_get_thread_name(thread),
@@ -1845,8 +1854,8 @@ void JvmtiExport::post_method_exit_inner(JavaThread* thread,
                                          bool exception_exit,
                                          frame current_frame,
                                          jvalue& value) {
-  if (mh->jvmti_mount_transition() || thread->is_in_VTMS_transition()) {
-    return; // no events should be posted if thread is in a VTMS transition
+  if (mh->jvmti_mount_transition() || thread->is_in_any_VTMS_transition()) {
+    return; // no events should be posted if thread is in any VTMS transition
   }
 
   EVT_TRIG_TRACE(JVMTI_EVENT_METHOD_EXIT, ("[%s] Trg Method Exit triggered %s.%s",
@@ -1921,8 +1930,8 @@ void JvmtiExport::post_single_step(JavaThread *thread, Method* method, address l
   if (state == NULL) {
     return;
   }
-  if (mh->jvmti_mount_transition() || thread->is_in_VTMS_transition()) {
-    return; // no events should be posted if thread is in a VTMS transition
+  if (mh->jvmti_mount_transition() || thread->is_in_any_VTMS_transition()) {
+    return; // no events should be posted if thread is in any VTMS transition
   }
 
   JvmtiEnvThreadStateIterator it(state);
@@ -1958,8 +1967,8 @@ void JvmtiExport::post_exception_throw(JavaThread *thread, Method* method, addre
   if (state == NULL) {
     return;
   }
-  if (thread->is_in_VTMS_transition()) {
-    return; // no events should be posted if thread is in a VTMS transition
+  if (thread->is_in_any_VTMS_transition()) {
+    return; // no events should be posted if thread is in any VTMS transition
   }
 
   EVT_TRIG_TRACE(JVMTI_EVENT_EXCEPTION, ("[%s] Trg Exception thrown triggered",
@@ -2080,8 +2089,8 @@ void JvmtiExport::notice_unwind_due_to_exception(JavaThread *thread, Method* met
       assert(!state->is_exception_caught(), "exception must not be caught yet.");
       state->set_exception_caught();
 
-      if (mh->jvmti_mount_transition() || thread->is_in_VTMS_transition()) {
-        return; // no events should be posted if thread is in a VTMS transition
+      if (mh->jvmti_mount_transition() || thread->is_in_any_VTMS_transition()) {
+        return; // no events should be posted if thread is in any VTMS transition
       }
       JvmtiEnvThreadStateIterator it(state);
       for (JvmtiEnvThreadState* ets = it.first(); ets != NULL; ets = it.next(ets)) {
@@ -2126,8 +2135,8 @@ void JvmtiExport::post_field_access_by_jni(JavaThread *thread, oop obj,
   // function don't make the call unless there is a Java context.
   assert(thread->has_last_Java_frame(), "must be called with a Java context");
 
-  if (thread->is_in_VTMS_transition()) {
-    return; // no events should be posted if thread is in a VTMS transition
+  if (thread->is_in_any_VTMS_transition()) {
+    return; // no events should be posted if thread is in any VTMS transition
   }
 
   ResourceMark rm;
@@ -2162,8 +2171,8 @@ void JvmtiExport::post_field_access(JavaThread *thread, Method* method,
   if (state == NULL) {
     return;
   }
-  if (thread->is_in_VTMS_transition()) {
-    return; // no events should be posted if thread is in a VTMS transition
+  if (thread->is_in_any_VTMS_transition()) {
+    return; // no events should be posted if thread is in any VTMS transition
   }
 
   EVT_TRIG_TRACE(JVMTI_EVENT_FIELD_ACCESS, ("[%s] Trg Field Access event triggered",
@@ -2212,8 +2221,8 @@ void JvmtiExport::post_field_modification_by_jni(JavaThread *thread, oop obj,
   // function don't make the call unless there is a Java context.
   assert(thread->has_last_Java_frame(), "must be called with Java context");
 
-  if (thread->is_in_VTMS_transition()) {
-    return; // no events should be posted if thread is in a VTMS transition
+  if (thread->is_in_any_VTMS_transition()) {
+    return; // no events should be posted if thread is in any VTMS transition
   }
 
   ResourceMark rm;
@@ -2243,8 +2252,8 @@ void JvmtiExport::post_raw_field_modification(JavaThread *thread, Method* method
   address location, Klass* field_klass, Handle object, jfieldID field,
   char sig_type, jvalue *value) {
 
-  if (thread->is_in_VTMS_transition()) {
-    return; // no events should be posted if thread is in a VTMS transition
+  if (thread->is_in_any_VTMS_transition()) {
+    return; // no events should be posted if thread is in any VTMS transition
   }
 
   if (sig_type == JVM_SIGNATURE_INT || sig_type == JVM_SIGNATURE_BOOLEAN ||
@@ -2318,8 +2327,8 @@ void JvmtiExport::post_field_modification(JavaThread *thread, Method* method,
   if (state == NULL) {
     return;
   }
-  if (thread->is_in_VTMS_transition()) {
-    return; // no events should be posted if thread is in a VTMS transition
+  if (thread->is_in_any_VTMS_transition()) {
+    return; // no events should be posted if thread is in any VTMS transition
   }
 
   EVT_TRIG_TRACE(JVMTI_EVENT_FIELD_MODIFICATION,
@@ -2357,8 +2366,8 @@ void JvmtiExport::post_native_method_bind(Method* method, address* function_ptr)
   HandleMark hm(thread);
   methodHandle mh(thread, method);
 
-  if (thread->is_in_VTMS_transition()) {
-    return; // no events should be posted if thread is in a VTMS transition
+  if (thread->is_in_any_VTMS_transition()) {
+    return; // no events should be posted if thread is in any VTMS transition
   }
   EVT_TRIG_TRACE(JVMTI_EVENT_NATIVE_METHOD_BIND, ("[%s] Trg Native Method Bind event triggered",
                       JvmtiTrace::safe_get_thread_name(thread)));
@@ -2431,7 +2440,7 @@ void JvmtiExport::post_compiled_method_load(nmethod *nm) {
   }
   JavaThread* thread = JavaThread::current();
 
-  assert(!thread->is_in_VTMS_transition(), "compiled method load events are not allowed in VTMS transition");
+  assert(!thread->is_in_any_VTMS_transition(), "compiled method load events are not allowed in any VTMS transition");
 
   EVT_TRIG_TRACE(JVMTI_EVENT_COMPILED_METHOD_LOAD,
                  ("[%s] method compile load event triggered",
@@ -2454,7 +2463,7 @@ void JvmtiExport::post_compiled_method_load(JvmtiEnv* env, nmethod *nm) {
   }
   JavaThread* thread = JavaThread::current();
 
-  assert(!thread->is_in_VTMS_transition(), "compiled method load events are not allowed in VTMS transition");
+  assert(!thread->is_in_any_VTMS_transition(), "compiled method load events are not allowed in any VTMS transition");
 
   EVT_TRACE(JVMTI_EVENT_COMPILED_METHOD_LOAD,
            ("[%s] method compile load event sent %s.%s  ",
@@ -2479,7 +2488,7 @@ void JvmtiExport::post_dynamic_code_generated_internal(const char *name, const v
 
   JavaThread* thread = JavaThread::current();
 
-  assert(!thread->is_in_VTMS_transition(), "dynamic code generated events are not allowed in VTMS transition");
+  assert(!thread->is_in_any_VTMS_transition(), "dynamic code generated events are not allowed in any VTMS transition");
 
   // In theory everyone coming thru here is in_vm but we need to be certain
   // because a callee will do a vm->native transition
@@ -2527,7 +2536,7 @@ void JvmtiExport::post_dynamic_code_generated(JvmtiEnv* env, const char *name,
 {
   JavaThread* thread = JavaThread::current();
 
-  assert(!thread->is_in_VTMS_transition(), "dynamic code generated events are not allowed in VTMS transition");
+  assert(!thread->is_in_any_VTMS_transition(), "dynamic code generated events are not allowed in any VTMS transition");
 
   EVT_TRIG_TRACE(JVMTI_EVENT_DYNAMIC_CODE_GENERATED,
                  ("[%s] dynamic code generated event triggered (by GenerateEvents)",
@@ -2551,7 +2560,7 @@ void JvmtiExport::post_dynamic_code_generated_while_holding_locks(const char* na
                                                                   address code_begin, address code_end)
 {
   JavaThread* thread = JavaThread::current();
-  assert(!thread->is_in_VTMS_transition(), "dynamic code generated events are not allowed in VTMS transition");
+  assert(!thread->is_in_any_VTMS_transition(), "dynamic code generated events are not allowed in any VTMS transition");
 
   // register the stub with the current dynamic code event collector
   // Cannot take safepoint here so do not use state_for to get
@@ -2681,8 +2690,8 @@ void JvmtiExport::post_monitor_contended_enter(JavaThread *thread, ObjectMonitor
   if (state == NULL) {
     return;
   }
-  if (thread->is_in_VTMS_transition()) {
-    return; // no events should be posted if thread is in a VTMS transition
+  if (thread->is_in_any_VTMS_transition()) {
+    return; // no events should be posted if thread is in any VTMS transition
   }
 
   HandleMark hm(thread);
@@ -2714,8 +2723,8 @@ void JvmtiExport::post_monitor_contended_entered(JavaThread *thread, ObjectMonit
   if (state == NULL) {
     return;
   }
-  if (thread->is_in_VTMS_transition()) {
-    return; // no events should be posted if thread is in a VTMS transition
+  if (thread->is_in_any_VTMS_transition()) {
+    return; // no events should be posted if thread is in any VTMS transition
   }
 
   HandleMark hm(thread);
@@ -2748,8 +2757,8 @@ void JvmtiExport::post_monitor_wait(JavaThread *thread, oop object,
   if (state == NULL) {
     return;
   }
-  if (thread->is_in_VTMS_transition()) {
-    return; // no events should be posted if thread is in a VTMS transition
+  if (thread->is_in_any_VTMS_transition()) {
+    return; // no events should be posted if thread is in any VTMS transition
   }
 
   HandleMark hm(thread);
@@ -2782,8 +2791,8 @@ void JvmtiExport::post_monitor_waited(JavaThread *thread, ObjectMonitor *obj_mnt
   if (state == NULL) {
     return;
   }
-  if (thread->is_in_VTMS_transition()) {
-    return; // no events should be posted if thread is in a VTMS transition
+  if (thread->is_in_any_VTMS_transition()) {
+    return; // no events should be posted if thread is in any VTMS transition
   }
 
   HandleMark hm(thread);
@@ -2814,8 +2823,8 @@ void JvmtiExport::post_vm_object_alloc(JavaThread *thread, oop object) {
   if (object == NULL) {
     return;
   }
-  if (thread->is_in_VTMS_transition()) {
-    return; // no events should be posted if thread is in a VTMS transition
+  if (thread->is_in_any_VTMS_transition()) {
+    return; // no events should be posted if thread is in any VTMS transition
   }
   HandleMark hm(thread);
   Handle h(thread, object);
@@ -2848,8 +2857,8 @@ void JvmtiExport::post_sampled_object_alloc(JavaThread *thread, oop object) {
   if (object == NULL) {
     return;
   }
-  if (thread->is_in_VTMS_transition()) {
-    return; // no events should be posted if thread is in a VTMS transition
+  if (thread->is_in_any_VTMS_transition()) {
+    return; // no events should be posted if thread is in any VTMS transition
   }
   HandleMark hm(thread);
   Handle h(thread, object);