[CI] ClusterDisruptionIT: Address already in use #29244
Description
https://elasticsearch-ci.elastic.co/job/elastic+elasticsearch+6.x+periodic/1703/console failed because it tried to bind to a port that's already in use:
java.lang.RuntimeException: failed to start nodes
at __randomizedtesting.SeedInfo.seed([1B588786F9BF83:C6650D14FA151162]:0)
at org.elasticsearch.test.InternalTestCluster.startAndPublishNodesAndClients(InternalTestCluster.java:1374)
at org.elasticsearch.test.InternalTestCluster.startNode(InternalTestCluster.java:1652)
at org.elasticsearch.test.InternalTestCluster.startDataOnlyNode(InternalTestCluster.java:1757)
at org.elasticsearch.test.InternalTestCluster.startDataOnlyNode(InternalTestCluster.java:1753)
at org.elasticsearch.discovery.ClusterDisruptionIT.testSearchWithRelocationAndSlowClusterStateProcessing(ClusterDisruptionIT.java:366)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:907)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:943)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:957)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:748)
Caused by: java.util.concurrent.ExecutionException: BindTransportException[Failed to bind to [30210]]; nested: BindException[Address already in use (Bind failed)];
at java.util.concurrent.FutureTask.report(FutureTask.java:122)
at java.util.concurrent.FutureTask.get(FutureTask.java:192)
at org.elasticsearch.test.InternalTestCluster.startAndPublishNodesAndClients(InternalTestCluster.java:1369)
... 41 more
Caused by: BindTransportException[Failed to bind to [30210]]; nested: BindException[Address already in use (Bind failed)];
at org.elasticsearch.transport.TcpTransport.bindToPort(TcpTransport.java:790)
at org.elasticsearch.transport.TcpTransport.bindServer(TcpTransport.java:755)
at org.elasticsearch.transport.MockTcpTransport.doStart(MockTcpTransport.java:407)
at org.elasticsearch.common.component.AbstractLifecycleComponent.start(AbstractLifecycleComponent.java:66)
at org.elasticsearch.test.transport.MockTransportService$DelegateTransport.start(MockTransportService.java:647)
at org.elasticsearch.transport.TransportService.doStart(TransportService.java:213)
at org.elasticsearch.common.component.AbstractLifecycleComponent.start(AbstractLifecycleComponent.java:66)
at org.elasticsearch.node.Node.start(Node.java:641)
at org.elasticsearch.test.InternalTestCluster$NodeAndClient.startNode(InternalTestCluster.java:854)
at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:573)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more
Caused by: java.net.BindException: Address already in use (Bind failed)
at java.net.PlainSocketImpl.socketBind(Native Method)
at java.net.AbstractPlainSocketImpl.bind(AbstractPlainSocketImpl.java:387)
at java.net.ServerSocket.bind(ServerSocket.java:375)
at java.net.ServerSocket.bind(ServerSocket.java:329)
at org.elasticsearch.mocksocket.MockServerSocket.access$001(MockServerSocket.java:32)
at org.elasticsearch.mocksocket.MockServerSocket.lambda$bind$1(MockServerSocket.java:63)
at java.security.AccessController.doPrivileged(Native Method)
at org.elasticsearch.mocksocket.MockServerSocket.bind(MockServerSocket.java:62)
at org.elasticsearch.transport.MockTcpTransport.bind(MockTcpTransport.java:120)
at org.elasticsearch.transport.MockTcpTransport.bind(MockTcpTransport.java:71)
at org.elasticsearch.transport.TcpTransport.lambda$bindToPort$16(TcpTransport.java:773)
at org.elasticsearch.common.transport.PortsRange.iterate(PortsRange.java:59)
at org.elasticsearch.transport.TcpTransport.bindToPort(TcpTransport.java:771)
... 14 more
It looks like a fix for this sort of thing went into #9527 but this doesn't seem to be in place any more.
FWIW the reproduction line was:
REPRODUCE WITH: ./gradlew :server:integTest \
-Dtests.seed=1B588786F9BF83 \
-Dtests.class=org.elasticsearch.discovery.ClusterDisruptionIT \
-Dtests.method="testSearchWithRelocationAndSlowClusterStateProcessing" \
-Dtests.security.manager=true \
-Dtests.locale=es-NI \
-Dtests.timezone=Pacific/Midway
I checked the worker and there's nothing binding to that port any more.