Open
Description
We have this logged a couple of times across various versions in Cloud (up to and including 8.10.1). There's some likely obvious concurrency bug with the unsynchronised use of an array list in this class combined with its use across multiple threads in the rest serialisation (and maybe elsewhere):
[instance-0000000003] An exception was thrown by org.elasticsearch.transport.netty4.Netty4TcpChannel$$Lambda$6393/0x000000e0024b24c8.operationComplete()
java.util.ConcurrentModificationException: null
at java.util.ArrayList$Itr.checkForComodification(ArrayList.java:1013) ~[?:?]
at java.util.ArrayList$Itr.next(ArrayList.java:967) ~[?:?]
at org.elasticsearch.core.IOUtils.close(IOUtils.java:117) ~[elasticsearch-core-8.9.0.jar:?]
at org.elasticsearch.core.Releasables.close(Releasables.java:24) ~[elasticsearch-core-8.9.0.jar:?]
at org.elasticsearch.common.io.stream.RecyclerBytesStreamOutput.close(RecyclerBytesStreamOutput.java:173) ~[elasticsearch-8.9.0.jar:?]
at org.elasticsearch.rest.AbstractRestChannel.releaseOutputBuffer(AbstractRestChannel.java:193) ~[elasticsearch-8.9.0.jar:?]
at org.elasticsearch.core.IOUtils.close(IOUtils.java:71) ~[elasticsearch-core-8.9.0.jar:?]
at org.elasticsearch.core.IOUtils.close(IOUtils.java:119) ~[elasticsearch-core-8.9.0.jar:?]
at org.elasticsearch.core.Releasables.close(Releasables.java:24) ~[elasticsearch-core-8.9.0.jar:?]
at org.elasticsearch.core.Releasables$1.close(Releasables.java:96) ~[elasticsearch-core-8.9.0.jar:?]
at org.elasticsearch.core.IOUtils.close(IOUtils.java:71) ~[elasticsearch-core-8.9.0.jar:?]
at org.elasticsearch.core.Releasables.close(Releasables.java:33) ~[elasticsearch-core-8.9.0.jar:?]
at org.elasticsearch.core.Releasables.closeExpectNoException(Releasables.java:57) ~[elasticsearch-core-8.9.0.jar:?]
at org.elasticsearch.action.ActionListenerImplementations$2.run(ActionListenerImplementations.java:47) ~[elasticsearch-8.9.0.jar:?]
at org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:120) ~[elasticsearch-8.9.0.jar:?]
at org.elasticsearch.transport.netty4.Netty4TcpChannel.lambda$addPromise$1(Netty4TcpChannel.java:78) ~[?:?]
at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[?:?]
at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:557) ~[?:?]
at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[?:?]
at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[?:?]
at io.netty.util.concurrent.DefaultPromise.setSuccess0(DefaultPromise.java:625) ~[?:?]
at io.netty.util.concurrent.DefaultPromise.trySuccess(DefaultPromise.java:105) ~[?:?]
at io.netty.util.concurrent.PromiseCombiner.tryPromise(PromiseCombiner.java:170) ~[?:?]
at io.netty.util.concurrent.PromiseCombiner.access$600(PromiseCombiner.java:35) ~[?:?]
at io.netty.util.concurrent.PromiseCombiner$1.operationComplete0(PromiseCombiner.java:62) ~[?:?]
at io.netty.util.concurrent.PromiseCombiner$1.operationComplete(PromiseCombiner.java:44) ~[?:?]
at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[?:?]
at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:557) ~[?:?]
at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[?:?]
at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[?:?]
at io.netty.util.concurrent.DefaultPromise.setSuccess0(DefaultPromise.java:625) ~[?:?]
at io.netty.util.concurrent.DefaultPromise.trySuccess(DefaultPromise.java:105) ~[?:?]
at io.netty.util.internal.PromiseNotificationUtil.trySuccess(PromiseNotificationUtil.java:48) ~[?:?]
at io.netty.channel.DelegatingChannelPromiseNotifier.operationComplete(DelegatingChannelPromiseNotifier.java:52) ~[?:?]
at io.netty.channel.DelegatingChannelPromiseNotifier.operationComplete(DelegatingChannelPromiseNotifier.java:31) ~[?:?]
at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[?:?]
at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[?:?]
at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[?:?]
at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[?:?]
at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[?:?]
at io.netty.util.concurrent.DefaultPromise.setSuccess0(DefaultPromise.java:625) ~[?:?]
at io.netty.util.concurrent.DefaultPromise.trySuccess(DefaultPromise.java:105) ~[?:?]
at io.netty.util.internal.PromiseNotificationUtil.trySuccess(PromiseNotificationUtil.java:48) ~[?:?]
at io.netty.channel.ChannelOutboundBuffer.safeSuccess(ChannelOutboundBuffer.java:728) ~[?:?]
at io.netty.channel.ChannelOutboundBuffer.remove(ChannelOutboundBuffer.java:283) ~[?:?]
at io.netty.channel.ChannelOutboundBuffer.removeBytes(ChannelOutboundBuffer.java:363) ~[?:?]
at org.elasticsearch.transport.netty4.CopyBytesSocketChannel.doWrite(CopyBytesSocketChannel.java:102) ~[?:?]
at io.netty.channel.AbstractChannel$AbstractUnsafe.flush0(AbstractChannel.java:931) ~[?:?]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.flush0(AbstractNioChannel.java:354) ~[?:?]
at io.netty.channel.AbstractChannel$AbstractUnsafe.flush(AbstractChannel.java:895) ~[?:?]
at io.netty.channel.DefaultChannelPipeline$HeadContext.flush(DefaultChannelPipeline.java:1372) ~[?:?]
at io.netty.channel.AbstractChannelHandlerContext.invokeFlush0(AbstractChannelHandlerContext.java:921) ~[?:?]
at io.netty.channel.AbstractChannelHandlerContext.invokeFlush(AbstractChannelHandlerContext.java:907) ~[?:?]
at io.netty.channel.AbstractChannelHandlerContext.flush(AbstractChannelHandlerContext.java:893) ~[?:?]
at io.netty.handler.ssl.SslHandler.forceFlush(SslHandler.java:2148) ~[?:?]
at io.netty.handler.ssl.SslHandler.wrapAndFlush(SslHandler.java:804) ~[?:?]
at io.netty.handler.ssl.SslHandler.flush(SslHandler.java:781) ~[?:?]
at io.netty.channel.AbstractChannelHandlerContext.invokeFlush0(AbstractChannelHandlerContext.java:925) ~[?:?]
at io.netty.channel.AbstractChannelHandlerContext.invokeFlush(AbstractChannelHandlerContext.java:907) ~[?:?]
at io.netty.channel.AbstractChannelHandlerContext.flush(AbstractChannelHandlerContext.java:893) ~[?:?]
at org.elasticsearch.transport.netty4.Netty4WriteThrottlingHandler.doFlush(Netty4WriteThrottlingHandler.java:146) ~[?:?]
at org.elasticsearch.transport.netty4.Netty4WriteThrottlingHandler.flush(Netty4WriteThrottlingHandler.java:101) ~[?:?]
at io.netty.channel.AbstractChannelHandlerContext.invokeFlush0(AbstractChannelHandlerContext.java:923) ~[?:?]
at io.netty.channel.AbstractChannelHandlerContext.invokeFlush(AbstractChannelHandlerContext.java:907) ~[?:?]
at io.netty.channel.AbstractChannelHandlerContext.flush(AbstractChannelHandlerContext.java:893) ~[?:?]
at org.elasticsearch.http.netty4.Netty4HttpPipeliningHandler.doFlush(Netty4HttpPipeliningHandler.java:305) ~[?:?]
at org.elasticsearch.http.netty4.Netty4HttpPipeliningHandler.flush(Netty4HttpPipeliningHandler.java:260) ~[?:?]
at io.netty.channel.AbstractChannelHandlerContext.invokeFlush0(AbstractChannelHandlerContext.java:923) ~[?:?]
at io.netty.channel.AbstractChannelHandlerContext.invokeWriteAndFlush(AbstractChannelHandlerContext.java:941) ~[?:?]
at io.netty.channel.AbstractChannelHandlerContext$WriteTask.run(AbstractChannelHandlerContext.java:1247) ~[?:?]
at io.netty.util.concurrent.AbstractEventExecutor.runTask(AbstractEventExecutor.java:174) ~[?:?]
at io.netty.util.concurrent.AbstractEventExecutor.safeExecute(AbstractEventExecutor.java:167) ~[?:?]
at io.netty.util.concurrent.SingleThreadEventExecutor.runAllTasks(SingleThreadEventExecutor.java:470) ~[?:?]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:569) ~[?:?]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[?:?]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[?:?]
at java.lang.Thread.run(Thread.java:1623) ~[?:?]
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment