JVMTI agent is not unloaded when Agent_OnAttach is failed

Author: YaSuenag

src/hotspot/share/prims/jvmtiExport.cpp

@@ -2641,7 +2641,9 @@ jint JvmtiExport::load_agent_library(const char *agent, const char *absParam,
   void* library = NULL;
   jint result = JNI_ERR;
   const char *on_attach_symbols[] = AGENT_ONATTACH_SYMBOLS;
-  size_t num_symbol_entries = ARRAY_SIZE(on_attach_symbols);
+  size_t num_onattach_symbol_entries = ARRAY_SIZE(on_attach_symbols);
+  const char *on_unload_symbols[] = AGENT_ONUNLOAD_SYMBOLS;
+  size_t num_onunload_symbol_entries = ARRAY_SIZE(on_unload_symbols);
 
   // The abs paramter should be "true" or "false"
   bool is_absolute_path = (absParam != NULL) && (strcmp(absParam,"true")==0);
@@ -2653,7 +2655,7 @@ jint JvmtiExport::load_agent_library(const char *agent, const char *absParam,
   // absolute we attempt to load the library. Otherwise we try to load it
   // from the standard dll directory.
 
-  if (!os::find_builtin_agent(agent_lib, on_attach_symbols, num_symbol_entries)) {
+  if (!os::find_builtin_agent(agent_lib, on_attach_symbols, num_onattach_symbol_entries)) {
     if (is_absolute_path) {
       library = os::dll_load(agent, ebuf, sizeof ebuf);
     } else {
@@ -2677,10 +2679,13 @@ jint JvmtiExport::load_agent_library(const char *agent, const char *absParam,
   // If the library was loaded then we attempt to invoke the Agent_OnAttach
   // function
   if (agent_lib->valid()) {
+    JavaThread* THREAD = JavaThread::current();
+    extern struct JavaVM_ main_vm;
+
     // Lookup the Agent_OnAttach function
     OnAttachEntry_t on_attach_entry = NULL;
     on_attach_entry = CAST_TO_FN_PTR(OnAttachEntry_t,
-       os::find_agent_function(agent_lib, false, on_attach_symbols, num_symbol_entries));
+       os::find_agent_function(agent_lib, false, on_attach_symbols, num_onattach_symbol_entries));
     if (on_attach_entry == NULL) {
       // Agent_OnAttach missing - unload library
       if (!agent_lib->is_static_lib()) {
@@ -2691,9 +2696,7 @@ jint JvmtiExport::load_agent_library(const char *agent, const char *absParam,
       delete agent_lib;
     } else {
       // Invoke the Agent_OnAttach function
-      JavaThread* THREAD = JavaThread::current();
       {
-        extern struct JavaVM_ main_vm;
         JvmtiThreadEventMark jem(THREAD);
         JvmtiJavaThreadEventTransition jet(THREAD);
 
@@ -2710,6 +2713,23 @@ jint JvmtiExport::load_agent_library(const char *agent, const char *absParam,
       if (result == JNI_OK) {
         Arguments::add_loaded_agent(agent_lib);
       } else {
+        // Find the Agent_OnUnload function.
+        Agent_OnUnload_t unload_entry = CAST_TO_FN_PTR(Agent_OnUnload_t,
+                                                       os::find_agent_function(agent_lib,
+                                                       false,
+                                                       on_unload_symbols,
+                                                       num_onunload_symbol_entries));
+        // Invoke the Agent_OnUnload function
+        if (unload_entry != NULL) {
+          JvmtiThreadEventMark jem(THREAD);
+          JvmtiJavaThreadEventTransition jet(THREAD);
+
+          (*unload_entry)(&main_vm);
+        }
+
+        if (!agent_lib->is_static_lib()) {
+          os::dll_unload(library);
+        }
         delete agent_lib;
       }
 

src/hotspot/share/prims/jvmtiExport.hpp

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 1998, 2019, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1998, 2020, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -40,6 +40,11 @@
 // Must be included after jvmti.h.
 #include "jvmticmlr.h"
 
+// type for the Agent_OnUnload entry point
+extern "C" {
+  typedef void (JNICALL *Agent_OnUnload_t)(JavaVM *);
+}
+
 // Forward declarations
 
 class JvmtiEventControllerPrivate;

src/hotspot/share/runtime/thread.cpp

@@ -4325,10 +4325,6 @@ void Threads::create_vm_init_agents() {
   JvmtiExport::enter_primordial_phase();
 }
 
-extern "C" {
-  typedef void (JNICALL *Agent_OnUnload_t)(JavaVM *);
-}
-
 void Threads::shutdown_vm_agents() {
   // Send any Agent_OnUnload notifications
   const char *on_unload_symbols[] = AGENT_ONUNLOAD_SYMBOLS;

test/hotspot/jtreg/serviceability/dcmd/jvmti/AttachFailed/AttachReturnError.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2017, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2017, 2020, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -25,7 +25,7 @@
 
 /*
  * @test
- * @bug 8165736
+ * @bug 8165736 8252657
  * @library /test/lib
  * @run testng AttachReturnError
  */
@@ -36,8 +36,13 @@ public void run(CommandExecutor executor)  {
             String libpath = getSharedObjectPath("ReturnError");
             OutputAnalyzer output = null;
 
+            // Check return code
             output = executor.execute("JVMTI.agent_load " + libpath);
             output.shouldContain("return code: -1");
+
+            // Check agent unloading
+            output = executor.execute("VM.dynlibs");
+            output.shouldNotContain(libpath);
         } catch (Exception e) {
             throw new RuntimeException(e);
         }