[MAPR-30583] InMemoryFileIndex changed to getFileBlockLocations in pa…

…rallel way (apache#221)
jamesrgrinter · Feb 7, 2018 · 2d9e466 · 2d9e466
1 parent 1b37b08
commit 2d9e466
Showing 1 changed file with 2 additions and 2 deletions.
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/InMemoryFileIndex.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/InMemoryFileIndex.scala
@@ -278,7 +278,7 @@ object InMemoryFileIndex extends Logging {
       if (filter != null) allFiles.filter(f => filter.accept(f.getPath)) else allFiles
     }
 
-    allLeafStatuses.filterNot(status => shouldFilterOut(status.getPath.getName)).map {
+    allLeafStatuses.par.filterNot(status => shouldFilterOut(status.getPath.getName)).map {
       case f: LocatedFileStatus =>
         f
 
@@ -302,7 +302,7 @@ object InMemoryFileIndex extends Logging {
           lfs.setSymlink(f.getSymlink)
         }
         lfs
-    }
+    }.seq
   }
 
   /** Checks if we should filter out this path name. */