Skip to content

Commit 76417f6

Browse files
committed
refine comments
1 parent c10af60 commit 76417f6

File tree

2 files changed

+6
-8
lines changed

2 files changed

+6
-8
lines changed

core/src/main/scala/org/apache/spark/SparkContext.scala

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -444,7 +444,7 @@ class SparkContext(config: SparkConf) extends Logging {
444444
* hdfs://a-hdfs-path/part-nnnnn
445445
* }}}
446446
*
447-
* Do `val rdd = sparkContext.wholeTextFile("hdfs://a-hdfs-path")`
447+
* Do `val rdd = sparkContext.wholeTextFile("hdfs://a-hdfs-path")`,
448448
*
449449
* <p> then `rdd` contains
450450
* {{{

core/src/main/scala/org/apache/spark/rdd/NewHadoopRDD.scala

Lines changed: 5 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -24,9 +24,13 @@ import org.apache.hadoop.conf.{Configurable, Configuration}
2424
import org.apache.hadoop.io.Writable
2525
import org.apache.hadoop.mapreduce._
2626

27-
import org.apache.spark.{InterruptibleIterator, Logging, Partition, SerializableWritable, SparkContext, TaskContext}
2827
import org.apache.spark.annotation.DeveloperApi
2928
import org.apache.spark.input.WholeTextFileInputFormat
29+
import org.apache.spark.InterruptibleIterator
30+
import org.apache.spark.Logging
31+
import org.apache.spark.Partition
32+
import org.apache.spark.SerializableWritable
33+
import org.apache.spark.{SparkContext, TaskContext}
3034

3135
private[spark] class NewHadoopPartition(
3236
rddId: Int,
@@ -82,13 +86,7 @@ class NewHadoopRDD[K, V](
8286
configurable.setConf(conf)
8387
case _ =>
8488
}
85-
8689
val jobContext = newJobContext(conf, jobId)
87-
88-
if (inputFormat.isInstanceOf[WholeTextFileInputFormat]) {
89-
inputFormat.asInstanceOf[WholeTextFileInputFormat].setMaxSplitSize(jobContext, minSplits)
90-
}
91-
9290
val rawSplits = inputFormat.getSplits(jobContext).toArray
9391
val result = new Array[Partition](rawSplits.size)
9492
for (i <- 0 until rawSplits.size) {

0 commit comments

Comments
 (0)