8252657: JVMTI agent is not unloaded when Agent_OnAttach is failed

src/hotspot/share/prims/jvmti.xml

@@ -686,9 +686,11 @@ Agent_OnUnload_L(JavaVM *vm)</example>
     The library will be unloaded (unless it is statically linked into the
     executable) and this function will be called if some platform specific
     mechanism causes the unload (an unload mechanism is not specified in this document)
-    or the library is (in effect) unloaded by the termination of the VM whether through
-    normal termination or VM failure, including start-up failure.
-    Uncontrolled shutdown is, of course, an exception to this rule.
+    or the library is (in effect) unloaded by the termination of the VM.
+    VM termination includes normal termination and VM failure, including start-up failure,
+    but not, of course, uncontrolled shutdown. An implementation may also
+    choose to not call this function if the <code>Agent_OnAttach</code>/
+    <code>Agent_OnAttach_L</code> function reported an error (returned a non-zero value).
     Note the distinction between this function and the
     <eventlink id="VMDeath">VM Death event</eventlink>: for the VM Death event
     to be sent, the VM must have run at least to the point of initialization and a valid

src/hotspot/share/prims/jvmtiExport.cpp

@@ -2700,6 +2700,9 @@ jint JvmtiExport::load_agent_library(const char *agent, const char *absParam,
       if (result == JNI_OK) {
         Arguments::add_loaded_agent(agent_lib);
       } else {
+        if (!agent_lib->is_static_lib()) {
+          os::dll_unload(library);
+        }
         delete agent_lib;
       }
 

test/hotspot/jtreg/serviceability/dcmd/jvmti/AttachFailed/AttachReturnError.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2017, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2017, 2020, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -25,7 +25,7 @@
 
 /*
  * @test
- * @bug 8165736
+ * @bug 8165736 8252657
  * @library /test/lib
  * @run testng AttachReturnError
  */
@@ -36,8 +36,13 @@ public void run(CommandExecutor executor)  {
             String libpath = getSharedObjectPath("ReturnError");
             OutputAnalyzer output = null;
 
+            // Check return code
             output = executor.execute("JVMTI.agent_load " + libpath);
             output.shouldContain("return code: -1");
+
+            // Check loaded libraries
+            output = executor.execute("VM.dynlibs");
+            output.shouldNotContain(libpath);
         } catch (Exception e) {
             throw new RuntimeException(e);
         }