Closed
Description
This has failed 3 times since the test was added roughly 12 hours ago in #47285, according to build-stats - once on each of master, 7.x, and 7.4.
Reproduce line (does not reproduce locally):
./gradlew ':server:integTest' --tests "org.elasticsearch.indices.state.CloseIndexIT.testRelocatedClosedIndexIssue" -Dtests.seed=5B0F32A483AA1A7F -Dtests.security.manager=true -Dtests.locale=nl-SX -Dtests.timezone=Pacific/Majuro -Dcompiler.java=12 -Druntime.java=11
Stack trace:
java.lang.NullPointerException
:
(No message provided)
Close stacktrace
at __randomizedtesting.SeedInfo.seed([5B0F32A483AA1A7F:842D8CFA6FEAA1AA]:0)
at org.elasticsearch.indices.state.CloseIndexIT.assertIndexIsClosed(CloseIndexIT.java:519)
at org.elasticsearch.indices.state.CloseIndexIT.testRelocatedClosedIndexIssue(CloseIndexIT.java:456)
at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:566)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1750)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:938)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:974)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:988)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:947)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:832)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:883)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:894)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:834)
Looks like an IndexMetaData that's expected to be non-null in this test is coming back null.
There's some stack traces in the logs which may or may not be relevant:
org.elasticsearch.cluster.coordination.FailedToCommitClusterStateException: node is no longer master for term 3 while handling publication
at org.elasticsearch.cluster.coordination.Coordinator.publish(Coordinator.java:1016) ~[main/:?]
at org.elasticsearch.cluster.service.MasterService.publish(MasterService.java:263) [main/:?]
at org.elasticsearch.cluster.service.MasterService.runTasks(MasterService.java:245) [main/:?]
at org.elasticsearch.cluster.service.MasterService$Batcher.run(MasterService.java:151) [main/:?]
at org.elasticsearch.cluster.service.TaskBatcher.runIfNotProcessed(TaskBatcher.java:150) [main/:?]
at org.elasticsearch.cluster.service.TaskBatcher$BatchedTask.run(TaskBatcher.java:188) [main/:?]
at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:699) [main/:?]
at org.elasticsearch.common.util.concurrent.PrioritizedEsThreadPoolExecutor$TieBreakingPrioritizedRunnable.runAndClean(PrioritizedEsThreadPoolExecutor.java:252) [main/:?]
at org.elasticsearch.common.util.concurrent.PrioritizedEsThreadPoolExecutor$TieBreakingPrioritizedRunnable.run(PrioritizedEsThreadPoolExecutor.java:215) [main/:?]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) [?:?]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) [?:?]
at java.lang.Thread.run(Thread.java:834) [?:?]
[2019-10-01T11:11:33,255][INFO ][o.e.c.s.MasterService ] [node_sm2] elected-as-master ([2] nodes joined)[{node_sm2}{AU4K0LeET0avsox4xjNQeQ}{uA-GaDB0Tq-dCRQUqSRLqA}{127.0.0.1}{127.0.0.1:45231}{im} elect leader, {node_sm1}{T5rEEHKbRLWh_CF6p1C2ZQ}{oy8MEjdySFqPBezYcF-kiQ}{127.0.0.1}{127.0.0.1:42041}{im} elect leader, _BECOME_MASTER_TASK_, _FINISH_ELECTION_], term: 3, version: 106, delta: master node changed {previous [], current [{node_sm2}{AU4K0LeET0avsox4xjNQeQ}{uA-GaDB0Tq-dCRQUqSRLqA}{127.0.0.1}{127.0.0.1:45231}{im}]}, added {{node_sm1}{T5rEEHKbRLWh_CF6p1C2ZQ}{oy8MEjdySFqPBezYcF-kiQ}{127.0.0.1}{127.0.0.1:42041}{im}}
[2019-10-01T11:11:33,438][INFO ][o.e.c.s.MasterService ] [node_sm1] elected-as-master ([2] nodes joined)[{node_sm0}{4nDa8zmcTE-3hnvJvZdCag}{rHe4feDtTEqQJ0_-RoJeVA}{127.0.0.1}{127.0.0.1:42605}{im} elect leader, {node_sm1}{T5rEEHKbRLWh_CF6p1C2ZQ}{oy8MEjdySFqPBezYcF-kiQ}{127.0.0.1}{127.0.0.1:42041}{im} elect leader, _BECOME_MASTER_TASK_, _FINISH_ELECTION_], term: 5, version: 107, delta: master node changed {previous [], current [{node_sm1}{T5rEEHKbRLWh_CF6p1C2ZQ}{oy8MEjdySFqPBezYcF-kiQ}{127.0.0.1}{127.0.0.1:42041}{im}]}, added {{node_sm0}{4nDa8zmcTE-3hnvJvZdCag}{rHe4feDtTEqQJ0_-RoJeVA}{127.0.0.1}{127.0.0.1:42605}{im}}
[2019-10-01T11:11:33,442][WARN ][o.e.c.s.MasterService ] [node_sm2] failing [elected-as-master ([2] nodes joined)[{node_sm2}{AU4K0LeET0avsox4xjNQeQ}{uA-GaDB0Tq-dCRQUqSRLqA}{127.0.0.1}{127.0.0.1:45231}{im} elect leader, {node_sm1}{T5rEEHKbRLWh_CF6p1C2ZQ}{oy8MEjdySFqPBezYcF-kiQ}{127.0.0.1}{127.0.0.1:42041}{im} elect leader, _BECOME_MASTER_TASK_, _FINISH_ELECTION_]]: failed to commit cluster state version [106]
org.elasticsearch.cluster.coordination.FailedToCommitClusterStateException: publication failed
at org.elasticsearch.cluster.coordination.Coordinator$CoordinatorPublication$4.onFailure(Coordinator.java:1393) ~[main/:?]
at org.elasticsearch.action.ActionRunnable.onFailure(ActionRunnable.java:60) ~[main/:?]
at org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:39) ~[main/:?]
at org.elasticsearch.common.util.concurrent.EsExecutors$DirectExecutorService.execute(EsExecutors.java:196) ~[main/:?]
at org.elasticsearch.common.util.concurrent.ListenableFuture.notifyListener(ListenableFuture.java:106) ~[main/:?]
at org.elasticsearch.common.util.concurrent.ListenableFuture.addListener(ListenableFuture.java:68) ~[main/:?]
at org.elasticsearch.cluster.coordination.Coordinator$CoordinatorPublication.onCompletion(Coordinator.java:1313) ~[main/:?]
at org.elasticsearch.cluster.coordination.Publication.onPossibleCompletion(Publication.java:125) ~[main/:?]
at org.elasticsearch.cluster.coordination.Publication.cancel(Publication.java:89) ~[main/:?]
at org.elasticsearch.cluster.coordination.Coordinator.cancelActivePublication(Coordinator.java:1093) ~[main/:?]
at org.elasticsearch.cluster.coordination.Coordinator.becomeCandidate(Coordinator.java:517) ~[main/:?]
at org.elasticsearch.cluster.coordination.Coordinator.joinLeaderInTerm(Coordinator.java:432) ~[main/:?]
at org.elasticsearch.cluster.coordination.JoinHelper.lambda$new$1(JoinHelper.java:129) ~[main/:?]
at org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:63) ~[main/:?]
at org.elasticsearch.transport.InboundHandler$RequestHandler.doRun(InboundHandler.java:262) ~[main/:?]
at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:769) ~[main/:?]
at org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) ~[main/:?]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) [?:?]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) [?:?]
at java.lang.Thread.run(Thread.java:834) [?:?]
Caused by: org.elasticsearch.ElasticsearchException: publication cancelled before committing: become candidate: joinLeaderInTerm
at org.elasticsearch.cluster.coordination.Publication.cancel(Publication.java:86) ~[main/:?]
... 11 more