Jdk connector dead lock

[jbescos]

This is part of the investigation of this issue #4810 but it does not fix the issue.

I found the next dead lock that is happening when ConnectionStateListener receives an error and the JDKConnector is closed. The lock 0x00000000cb635a68 corresponds with DestinationConnectionPool#removeAllPendingWithError. I don't see a reason to lock that method and it is causing the dead lock.

"main" #1 prio=5 os_prio=0 cpu=1737.72ms elapsed=161.43s tid=0x00007fc400012000 nid=0x1e46 waiting for monitor entry  [0x00007fc4086e0000]
   java.lang.Thread.State: BLOCKED (on object monitor)
	at org.glassfish.jersey.jdk.connector.internal.SslFilter.close(SslFilter.java:244)
	- waiting to lock <0x00000000cb546398> (a org.glassfish.jersey.jdk.connector.internal.SslFilter)
	at org.glassfish.jersey.jdk.connector.internal.Filter.close(Filter.java:58)
	at org.glassfish.jersey.jdk.connector.internal.Filter.close(Filter.java:58)
	at org.glassfish.jersey.jdk.connector.internal.HttpConnection.close(HttpConnection.java:136)
	at org.glassfish.jersey.jdk.connector.internal.DestinationConnectionPool$$Lambda$313/0x0000000100302c40.accept(Unknown Source)
	at java.util.concurrent.ConcurrentHashMap$KeySetView.forEach(java.base@11/ConcurrentHashMap.java:4698)
	at java.util.Collections$SetFromMap.forEach(java.base@11/Collections.java:5527)
	at org.glassfish.jersey.jdk.connector.internal.DestinationConnectionPool.close(DestinationConnectionPool.java:150)
	- locked <0x00000000cb635a68> (a org.glassfish.jersey.jdk.connector.internal.DestinationConnectionPool)
	at org.glassfish.jersey.jdk.connector.internal.HttpConnectionPool$$Lambda$312/0x0000000100302840.accept(Unknown Source)
	at java.util.concurrent.ConcurrentHashMap$ValuesView.forEach(java.base@11/ConcurrentHashMap.java:4772)
	at org.glassfish.jersey.jdk.connector.internal.HttpConnectionPool.close(HttpConnectionPool.java:78)
	- locked <0x00000000cb583670> (a org.glassfish.jersey.jdk.connector.internal.HttpConnectionPool)
	at org.glassfish.jersey.jdk.connector.internal.JdkConnector.close(JdkConnector.java:246)
	at org.glassfish.jersey.client.ClientRuntime.close(ClientRuntime.java:368)
	at org.glassfish.jersey.client.ClientRuntime.onShutdown(ClientRuntime.java:353)
	at org.glassfish.jersey.client.JerseyClient.release(JerseyClient.java:190)
	at org.glassfish.jersey.client.JerseyClient.close(JerseyClient.java:180)
	at org.glassfish.jersey.test.JerseyTest.closeIfNotNull(JerseyTest.java:953)
	at org.glassfish.jersey.test.JerseyTest.tearDown(JerseyTest.java:648)
	at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(java.base@11/Native Method)
	at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(java.base@11/NativeMethodAccessorImpl.java:62)
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(java.base@11/DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(java.base@11/Method.java:566)
	at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
	at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
	at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
	at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46)
	at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33)
	at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
	at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
	at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
	at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
	at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
	at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
	at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
	at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
	at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
	at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
	at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
	at org.junit.runners.ParentRunner.run(ParentRunner.java:413)
	at org.apache.maven.surefire.junit4.JUnit4Provider.execute(JUnit4Provider.java:364)
	at org.apache.maven.surefire.junit4.JUnit4Provider.executeWithRerun(JUnit4Provider.java:272)
	at org.apache.maven.surefire.junit4.JUnit4Provider.executeTestSet(JUnit4Provider.java:237)
	at org.apache.maven.surefire.junit4.JUnit4Provider.invoke(JUnit4Provider.java:158)
	at org.apache.maven.surefire.booter.ForkedBooter.runSuitesInProcess(ForkedBooter.java:428)
	at org.apache.maven.surefire.booter.ForkedBooter.execute(ForkedBooter.java:162)
	at org.apache.maven.surefire.booter.ForkedBooter.run(ForkedBooter.java:562)
	at org.apache.maven.surefire.booter.ForkedBooter.main(ForkedBooter.java:548)


"jdk-connector-5" #56 daemon prio=5 os_prio=0 cpu=43.65ms elapsed=159.17s tid=0x00007fc398096000 nid=0x1e8d waiting for monitor entry  [0x00007fc37d740000]
   java.lang.Thread.State: BLOCKED (on object monitor)
	at org.glassfish.jersey.jdk.connector.internal.DestinationConnectionPool.removeAllPendingWithError(DestinationConnectionPool.java:199)
	- waiting to lock <0x00000000cb635a68> (a org.glassfish.jersey.jdk.connector.internal.DestinationConnectionPool)
	at org.glassfish.jersey.jdk.connector.internal.DestinationConnectionPool.access$900(DestinationConnectionPool.java:34)
	at org.glassfish.jersey.jdk.connector.internal.DestinationConnectionPool$ConnectionStateListener.onStateChanged(DestinationConnectionPool.java:282)
	at org.glassfish.jersey.jdk.connector.internal.HttpConnection.changeState(HttpConnection.java:231)
	at org.glassfish.jersey.jdk.connector.internal.HttpConnection.handleError(HttpConnection.java:317)
	- locked <0x00000000cb546320> (a org.glassfish.jersey.jdk.connector.internal.HttpConnection)
	at org.glassfish.jersey.jdk.connector.internal.HttpConnection.access$000(HttpConnection.java:39)
	at org.glassfish.jersey.jdk.connector.internal.HttpConnection$ConnectionFilter.processError(HttpConnection.java:429)
	at org.glassfish.jersey.jdk.connector.internal.Filter.onError(Filter.java:165)
	at org.glassfish.jersey.jdk.connector.internal.Filter.onError(Filter.java:168)
	at org.glassfish.jersey.jdk.connector.internal.Filter.onError(Filter.java:168)
	at org.glassfish.jersey.jdk.connector.internal.SslFilter.handleSslError(SslFilter.java:568)
	at org.glassfish.jersey.jdk.connector.internal.SslFilter.doHandshakeStep(SslFilter.java:519)
	- locked <0x00000000cb546398> (a org.glassfish.jersey.jdk.connector.internal.SslFilter)
	at org.glassfish.jersey.jdk.connector.internal.SslFilter.processRead(SslFilter.java:314)
	at org.glassfish.jersey.jdk.connector.internal.SslFilter.processRead(SslFilter.java:35)
	at org.glassfish.jersey.jdk.connector.internal.Filter.onRead(Filter.java:116)
	at org.glassfish.jersey.jdk.connector.internal.Filter.onRead(Filter.java:124)
	at org.glassfish.jersey.jdk.connector.internal.TransportFilter$3.completed(TransportFilter.java:262)
	at org.glassfish.jersey.jdk.connector.internal.TransportFilter$3.completed(TransportFilter.java:249)
	at sun.nio.ch.Invoker.invokeUnchecked(java.base@11/Invoker.java:127)
	at sun.nio.ch.Invoker$2.run(java.base@11/Invoker.java:219)
	at sun.nio.ch.AsynchronousChannelGroupImpl$1.run(java.base@11/AsynchronousChannelGroupImpl.java:112)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@11/ThreadPoolExecutor.java:1128)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@11/ThreadPoolExecutor.java:628)
	at java.lang.Thread.run(java.base@11/Thread.java:834)

[senivam]

LGTM