[BUG] FileSystem Closed error occurs after restarting the cluster

[whutpencil]

I use the trino 408 version, and cherry pick this: #17869

and almost every time I restart cluster, the first round of queries will report the FileSystem Close error.

During the daily query process, there may also be occasional FileSystem Closed errors.

io.trino.spi.TrinoException: Filesystem closed
	at io.trino.plugin.hive.GenericHiveRecordCursor.advanceNextPosition(GenericHiveRecordCursor.java:233)
	at io.trino.plugin.hive.util.ForwardingRecordCursor.advanceNextPosition(ForwardingRecordCursor.java:46)
	at io.trino.$gen.CursorProcessor_20231228_070300_2369.process(Unknown Source)
	at io.trino.operator.ScanFilterAndProjectOperator$RecordCursorToPages.process(ScanFilterAndProjectOperator.java:335)
	at io.trino.operator.WorkProcessorUtils$ProcessWorkProcessor.process(WorkProcessorUtils.java:413)
	at io.trino.operator.WorkProcessorUtils.getNextState(WorkProcessorUtils.java:262)
	at io.trino.operator.WorkProcessorUtils$YieldingProcess.process(WorkProcessorUtils.java:182)
	at io.trino.operator.WorkProcessorUtils$ProcessWorkProcessor.process(WorkProcessorUtils.java:413)
	at io.trino.operator.WorkProcessorUtils.getNextState(WorkProcessorUtils.java:262)
	at io.trino.operator.WorkProcessorUtils$BlockingProcess.process(WorkProcessorUtils.java:208)
	at io.trino.operator.WorkProcessorUtils$ProcessWorkProcessor.process(WorkProcessorUtils.java:413)
	at io.trino.operator.WorkProcessorUtils.lambda$flatten$6(WorkProcessorUtils.java:318)
	at io.trino.operator.WorkProcessorUtils$3.process(WorkProcessorUtils.java:360)
	at io.trino.operator.WorkProcessorUtils$ProcessWorkProcessor.process(WorkProcessorUtils.java:413)
	at io.trino.operator.WorkProcessorUtils$3.process(WorkProcessorUtils.java:347)
	at io.trino.operator.WorkProcessorUtils$ProcessWorkProcessor.process(WorkProcessorUtils.java:413)
	at io.trino.operator.WorkProcessorUtils.getNextState(WorkProcessorUtils.java:262)
	at io.trino.operator.WorkProcessorUtils.lambda$processStateMonitor$2(WorkProcessorUtils.java:241)
	at io.trino.operator.WorkProcessorUtils$ProcessWorkProcessor.process(WorkProcessorUtils.java:413)
	at io.trino.operator.WorkProcessorUtils.getNextState(WorkProcessorUtils.java:262)
	at io.trino.operator.WorkProcessorUtils.lambda$finishWhen$3(WorkProcessorUtils.java:256)
	at io.trino.operator.WorkProcessorUtils$ProcessWorkProcessor.process(WorkProcessorUtils.java:413)
	at io.trino.operator.WorkProcessorSourceOperatorAdapter.getOutput(WorkProcessorSourceOperatorAdapter.java:146)
	at io.trino.operator.Driver.processInternal(Driver.java:394)
	at io.trino.operator.Driver.lambda$process$8(Driver.java:297)
	at io.trino.operator.Driver.tryWithLock(Driver.java:689)
	at io.trino.operator.Driver.process(Driver.java:289)
	at io.trino.operator.Driver.processForDuration(Driver.java:260)
	at io.trino.execution.SqlTaskExecution$DriverSplitRunner.processFor(SqlTaskExecution.java:773)
	at io.trino.execution.executor.PrioritizedSplitRunner.process(PrioritizedSplitRunner.java:165)
	at io.trino.execution.executor.TaskExecutor$TaskRunner.run(TaskExecutor.java:523)
	at io.trino.$gen.Trino_408_4fcc08ee_2023_12_10T20_30_38____20231227_113914_2.run(Unknown Source)
	at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
	at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	at java.base/java.lang.Thread.run(Thread.java:833)
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:494)
	at org.apache.hadoop.hdfs.DFSInputStream.readWithStrategy(DFSInputStream.java:827)
	at org.apache.hadoop.hdfs.DFSInputStream.read(DFSInputStream.java:918)
	at java.base/java.io.DataInputStream.read(DataInputStream.java:151)
	at org.apache.hadoop.mapreduce.lib.input.UncompressedSplitLineReader.fillBuffer(UncompressedSplitLineReader.java:62)
	at org.apache.hadoop.util.LineReader.readDefaultLine(LineReader.java:234)
	at org.apache.hadoop.util.LineReader.readLine(LineReader.java:192)
	at org.apache.hadoop.mapreduce.lib.input.UncompressedSplitLineReader.readLine(UncompressedSplitLineReader.java:94)
	at org.apache.hadoop.mapred.LineRecordReader.skipUtfByteOrderMark(LineRecordReader.java:228)
	at org.apache.hadoop.mapred.LineRecordReader.next(LineRecordReader.java:266)
	at org.apache.hadoop.mapred.LineRecordReader.next(LineRecordReader.java:53)
	at io.trino.plugin.hive.GenericHiveRecordCursor.advanceNextPosition(GenericHiveRecordCursor.java:215)
	... 34 more
	Suppressed: java.io.UncheckedIOException: java.io.IOException: Filesystem closed
		at io.trino.plugin.hive.GenericHiveRecordCursor.close(GenericHiveRecordCursor.java:585)
		at io.trino.plugin.base.util.Closables.closeAllSuppress(Closables.java:29)
		at io.trino.plugin.hive.GenericHiveRecordCursor.advanceNextPosition(GenericHiveRecordCursor.java:229)
		... 34 more
	Caused by: java.io.IOException: Filesystem closed
		at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:494)
		at org.apache.hadoop.hdfs.DFSInputStream.close(DFSInputStream.java:728)
		at java.base/java.io.FilterInputStream.close(FilterInputStream.java:179)
		at org.apache.hadoop.util.LineReader.close(LineReader.java:157)
		at org.apache.hadoop.mapred.LineRecordReader.close(LineRecordReader.java:304)
		at io.trino.plugin.hive.GenericHiveRecordCursor.close(GenericHiveRecordCursor.java:582)
		... 36 more

[whutpencil]

@electrum @findepi PTAL

[s-unscrupulous]

I am encountering the same issue, have you managed to resolve it yet? @whutpencil