Skip to content

Commit 36940df

Browse files
committed
Close cache for local mode
1 parent 935fed6 commit 36940df

File tree

1 file changed

+4
-2
lines changed

1 file changed

+4
-2
lines changed

core/src/main/scala/org/apache/spark/executor/Executor.scala

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -322,14 +322,16 @@ private[spark] class Executor(
322322
// Fetch missing dependencies
323323
for ((name, timestamp) <- newFiles if currentFiles.getOrElse(name, -1L) < timestamp) {
324324
logInfo("Fetching " + name + " with timestamp " + timestamp)
325+
// Fetch file with useCache mode, close cache for local mode.
325326
Utils.fetchFile(name, new File(SparkFiles.getRootDirectory), conf,
326-
env.securityManager, hadoopConf, timestamp, useCache = true)
327+
env.securityManager, hadoopConf, timestamp, useCache = !isLocal)
327328
currentFiles(name) = timestamp
328329
}
329330
for ((name, timestamp) <- newJars if currentJars.getOrElse(name, -1L) < timestamp) {
330331
logInfo("Fetching " + name + " with timestamp " + timestamp)
332+
// Fetch file with useCache mode, close cache for local mode.
331333
Utils.fetchFile(name, new File(SparkFiles.getRootDirectory), conf,
332-
env.securityManager, hadoopConf, timestamp, useCache = true)
334+
env.securityManager, hadoopConf, timestamp, useCache = !isLocal)
333335
currentJars(name) = timestamp
334336
// Add it to our class loader
335337
val localName = name.split("/").last

0 commit comments

Comments
 (0)