[MAPR-30583] InMemoryFileIndex changed to getFileBlockLocations in parallel way (apache#221)

rsotn-mapr · mgorbov · commit 2d9e46661171 · 2018-02-07T15:40:40.000+02:00
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/InMemoryFileIndex.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/InMemoryFileIndex.scala
@@ -278,7 +278,7 @@ object InMemoryFileIndex extends Logging {
       if (filter != null) allFiles.filter(f => filter.accept(f.getPath)) else allFiles
     }
 
-    allLeafStatuses.filterNot(status => shouldFilterOut(status.getPath.getName)).map {
+    allLeafStatuses.par.filterNot(status => shouldFilterOut(status.getPath.getName)).map {
       case f: LocatedFileStatus =>
         f
 
@@ -302,7 +302,7 @@ object InMemoryFileIndex extends Logging {
           lfs.setSymlink(f.getSymlink)
         }
         lfs
-    }
+    }.seq
   }
 
   /** Checks if we should filter out this path name. */

Original file line number	Diff line number	Diff line change
`@@ -278,7 +278,7 @@ object InMemoryFileIndex extends Logging {`
`278`	`278`	`if (filter != null) allFiles.filter(f => filter.accept(f.getPath)) else allFiles`
`279`	`279`	`}`
`280`	`280`
`281`		`- allLeafStatuses.filterNot(status => shouldFilterOut(status.getPath.getName)).map {`
	`281`	`+ allLeafStatuses.par.filterNot(status => shouldFilterOut(status.getPath.getName)).map {`
`282`	`282`	`case f: LocatedFileStatus =>`
`283`	`283`	`f`
`284`	`284`
`@@ -302,7 +302,7 @@ object InMemoryFileIndex extends Logging {`
`302`	`302`	`lfs.setSymlink(f.getSymlink)`
`303`	`303`	`}`
`304`	`304`	`lfs`
`305`		`- }`
	`305`	`+ }.seq`
`306`	`306`	`}`
`307`	`307`
`308`	`308`	`/** Checks if we should filter out this path name. */`