File tree Expand file tree Collapse file tree 1 file changed +5
-5
lines changed
sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/parquet Expand file tree Collapse file tree 1 file changed +5
-5
lines changed Original file line number Diff line number Diff line change @@ -479,9 +479,9 @@ object ParquetFileFormat extends Logging {
479
479
partFiles : Seq [FileStatus ],
480
480
ignoreCorruptFiles : Boolean ): Seq [Footer ] = {
481
481
val parFiles = partFiles.par
482
- val readParquetTaskSupport = new ForkJoinTaskSupport ( new ForkJoinPool (8 ) )
483
- parFiles.tasksupport = readParquetTaskSupport
484
- val footers = parFiles.flatMap { currentFile =>
482
+ val pool = new ForkJoinPool (8 )
483
+ parFiles.tasksupport = new ForkJoinTaskSupport (pool)
484
+ parFiles.flatMap { currentFile =>
485
485
try {
486
486
// Skips row group information since we only need the schema.
487
487
// ParquetFileReader.readFooter throws RuntimeException, instead of IOException,
@@ -496,10 +496,10 @@ object ParquetFileFormat extends Logging {
496
496
} else {
497
497
throw new IOException (s " Could not read footer for file: $currentFile" , e)
498
498
}
499
+ } finally {
500
+ pool.shutdown()
499
501
}
500
502
}.seq
501
- readParquetTaskSupport.forkJoinPool.shutdown()
502
- footers
503
503
}
504
504
505
505
/**
You can’t perform that action at this time.
0 commit comments