Skip to content

Commit 6bb725e

Browse files
author
liguoqiang
committed
fix #SPARK-1149 Bad partitioners can cause Spark to hang
1 parent edf8a56 commit 6bb725e

File tree

1 file changed

+2
-0
lines changed

1 file changed

+2
-0
lines changed

core/src/main/scala/org/apache/spark/SparkContext.scala

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -847,6 +847,8 @@ class SparkContext(
847847
partitions: Seq[Int],
848848
allowLocal: Boolean,
849849
resultHandler: (Int, U) => Unit) {
850+
val rddPartitions = rdd.partitions.map(_.index)
851+
partitions.foreach(p =>require(rddPartitions.contains(p), "partition index out of range: " +p))
850852
val callSite = getCallSite
851853
val cleanedFunc = clean(func)
852854
logInfo("Starting job: " + callSite)

0 commit comments

Comments
 (0)