Update patch

Author: YaSuenag

src/hotspot/share/prims/jvmti.xml

@@ -688,7 +688,10 @@ Agent_OnUnload_L(JavaVM *vm)</example>
     mechanism causes the unload (an unload mechanism is not specified in this document)
     or the library is (in effect) unloaded by the termination of the VM whether through
     normal termination or VM failure, including start-up failure.
-    Uncontrolled shutdown is, of course, an exception to this rule.
+    Uncontrolled shutdown and Agent Start-Up failure
+    (<internallink id="onload"><code>Agent_OnLoad</code></internallink> and
+    <internallink id="onattach"><code>Agent_OnAttach</code></internallink>) are, of course,
+    an exception to this rule.
     Note the distinction between this function and the
     <eventlink id="VMDeath">VM Death event</eventlink>: for the VM Death event
     to be sent, the VM must have run at least to the point of initialization and a valid

src/hotspot/share/prims/jvmtiExport.cpp

@@ -2733,6 +2733,9 @@ jint JvmtiExport::load_agent_library(const char *agent, const char *absParam,
       if (result == JNI_OK) {
         Arguments::add_loaded_agent(agent_lib);
       } else {
+        if (!agent_lib->is_static_lib()) {
+          os::dll_unload(library);
+        }
         delete agent_lib;
       }
 

test/hotspot/jtreg/serviceability/dcmd/jvmti/AttachFailed/AttachReturnError.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2017, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2017, 2020, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -25,7 +25,7 @@
 
 /*
  * @test
- * @bug 8165736
+ * @bug 8165736 8252657
  * @library /test/lib
  * @run testng AttachReturnError
  */
@@ -36,8 +36,13 @@ public void run(CommandExecutor executor)  {
             String libpath = getSharedObjectPath("ReturnError");
             OutputAnalyzer output = null;
 
+            // Check return code
             output = executor.execute("JVMTI.agent_load " + libpath);
             output.shouldContain("return code: -1");
+
+            // Check loaded libraries
+            output = executor.execute("VM.dynlibs");
+            output.shouldNotContain(libpath);
         } catch (Exception e) {
             throw new RuntimeException(e);
         }