[Spark 1877] ClassNotFoundException when loading RDD with serialized objects

tdas · tdas · commit 52eb54d02403 · 2014-05-19T22:36:24.000-07:00
Updated version of #821 Author: Tathagata Das <tathagata.das1565@gmail.com> Author: Ghidireac <bogdang@u448a5b0a73d45358d94a.ant.amazon.com> Closes #835 from tdas/SPARK-1877 and squashes the following commits: f346f71 [Tathagata Das] Addressed Patrick's comments. fee0c5d [Ghidireac] SPARK-1877: ClassNotFoundException when loading RDD with serialized objects
diff --git a/core/src/main/scala/org/apache/spark/SparkContext.scala b/core/src/main/scala/org/apache/spark/SparkContext.scala
@@ -718,7 +718,7 @@ class SparkContext(config: SparkConf) extends Logging {
       minPartitions: Int = defaultMinPartitions
       ): RDD[T] = {
     sequenceFile(path, classOf[NullWritable], classOf[BytesWritable], minPartitions)
-      .flatMap(x => Utils.deserialize[Array[T]](x._2.getBytes))
+      .flatMap(x => Utils.deserialize[Array[T]](x._2.getBytes, Utils.getContextOrSparkClassLoader))
   }
 
   protected[spark] def checkpointFile[T: ClassTag](

Original file line number	Diff line number	Diff line change
`@@ -718,7 +718,7 @@ class SparkContext(config: SparkConf) extends Logging {`
`718`	`718`	`minPartitions: Int = defaultMinPartitions`
`719`	`719`	`): RDD[T] = {`
`720`	`720`	`sequenceFile(path, classOf[NullWritable], classOf[BytesWritable], minPartitions)`
`721`		`- .flatMap(x => Utils.deserialize[Array[T]](x._2.getBytes))`
	`721`	`+ .flatMap(x => Utils.deserialize[Array[T]](x._2.getBytes, Utils.getContextOrSparkClassLoader))`
`722`	`722`	`}`
`723`	`723`
`724`	`724`	`protected[spark] def checkpointFile[T: ClassTag](`