Skip to content

Commit 14e09aa

Browse files
committed
Address comment.
1 parent e4940b9 commit 14e09aa

File tree

1 file changed

+5
-5
lines changed

1 file changed

+5
-5
lines changed

sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetFileFormat.scala

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -479,9 +479,9 @@ object ParquetFileFormat extends Logging {
479479
partFiles: Seq[FileStatus],
480480
ignoreCorruptFiles: Boolean): Seq[Footer] = {
481481
val parFiles = partFiles.par
482-
val readParquetTaskSupport = new ForkJoinTaskSupport(new ForkJoinPool(8))
483-
parFiles.tasksupport = readParquetTaskSupport
484-
val footers = parFiles.flatMap { currentFile =>
482+
val pool = new ForkJoinPool(8)
483+
parFiles.tasksupport = new ForkJoinTaskSupport(pool)
484+
parFiles.flatMap { currentFile =>
485485
try {
486486
// Skips row group information since we only need the schema.
487487
// ParquetFileReader.readFooter throws RuntimeException, instead of IOException,
@@ -496,10 +496,10 @@ object ParquetFileFormat extends Logging {
496496
} else {
497497
throw new IOException(s"Could not read footer for file: $currentFile", e)
498498
}
499+
} finally {
500+
pool.shutdown()
499501
}
500502
}.seq
501-
readParquetTaskSupport.forkJoinPool.shutdown()
502-
footers
503503
}
504504

505505
/**

0 commit comments

Comments
 (0)