Skip to content

Commit 12bbca2

Browse files
srowenpwendell
srowen
authored and
pwendell
committed
SPARK 1084.1 (resubmitted)
(Ported from https://github.com/apache/incubator-spark/pull/637 ) Author: Sean Owen <sowen@cloudera.com> Closes #31 from srowen/SPARK-1084.1 and squashes the following commits: 6c4a32c [Sean Owen] Suppress warnings about legitimate unchecked array creations, or change code to avoid it f35b833 [Sean Owen] Fix two misc javadoc problems 254e8ef [Sean Owen] Fix one new style error introduced in scaladoc warning commit 5b2fce2 [Sean Owen] Fix scaladoc invocation warning, and enable javac warnings properly, with plugin config updates 007762b [Sean Owen] Remove dead scaladoc links b8ff8cb [Sean Owen] Replace deprecated Ant <tasks> with <target>
1 parent aace2c0 commit 12bbca2

File tree

15 files changed

+154
-88
lines changed

15 files changed

+154
-88
lines changed

‎bagel/src/main/scala/org/apache/spark/bagel/Bagel.scala

Lines changed: 7 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -27,7 +27,7 @@ object Bagel extends Logging {
2727

2828
/**
2929
* Runs a Bagel program.
30-
* @param sc [[org.apache.spark.SparkContext]] to use for the program.
30+
* @param sc org.apache.spark.SparkContext to use for the program.
3131
* @param vertices vertices of the graph represented as an RDD of (Key, Vertex) pairs. Often the
3232
* Key will be the vertex id.
3333
* @param messages initial set of messages represented as an RDD of (Key, Message) pairs. Often
@@ -38,10 +38,10 @@ object Bagel extends Logging {
3838
* @param aggregator [[org.apache.spark.bagel.Aggregator]] performs a reduce across all vertices
3939
* after each superstep and provides the result to each vertex in the next
4040
* superstep.
41-
* @param partitioner [[org.apache.spark.Partitioner]] partitions values by key
41+
* @param partitioner org.apache.spark.Partitioner partitions values by key
4242
* @param numPartitions number of partitions across which to split the graph.
4343
* Default is the default parallelism of the SparkContext
44-
* @param storageLevel [[org.apache.spark.storage.StorageLevel]] to use for caching of
44+
* @param storageLevel org.apache.spark.storage.StorageLevel to use for caching of
4545
* intermediate RDDs in each superstep. Defaults to caching in memory.
4646
* @param compute function that takes a Vertex, optional set of (possibly combined) messages to
4747
* the Vertex, optional Aggregator and the current superstep,
@@ -131,7 +131,7 @@ object Bagel extends Logging {
131131

132132
/**
133133
* Runs a Bagel program with no [[org.apache.spark.bagel.Aggregator]], default
134-
* [[org.apache.spark.HashPartitioner]] and default storage level
134+
* org.apache.spark.HashPartitioner and default storage level
135135
*/
136136
def run[K: Manifest, V <: Vertex : Manifest, M <: Message[K] : Manifest, C: Manifest](
137137
sc: SparkContext,
@@ -146,7 +146,7 @@ object Bagel extends Logging {
146146

147147
/**
148148
* Runs a Bagel program with no [[org.apache.spark.bagel.Aggregator]] and the
149-
* default [[org.apache.spark.HashPartitioner]]
149+
* default org.apache.spark.HashPartitioner
150150
*/
151151
def run[K: Manifest, V <: Vertex : Manifest, M <: Message[K] : Manifest, C: Manifest](
152152
sc: SparkContext,
@@ -166,7 +166,7 @@ object Bagel extends Logging {
166166

167167
/**
168168
* Runs a Bagel program with no [[org.apache.spark.bagel.Aggregator]],
169-
* default [[org.apache.spark.HashPartitioner]],
169+
* default org.apache.spark.HashPartitioner,
170170
* [[org.apache.spark.bagel.DefaultCombiner]] and the default storage level
171171
*/
172172
def run[K: Manifest, V <: Vertex : Manifest, M <: Message[K] : Manifest](
@@ -180,7 +180,7 @@ object Bagel extends Logging {
180180

181181
/**
182182
* Runs a Bagel program with no [[org.apache.spark.bagel.Aggregator]],
183-
* the default [[org.apache.spark.HashPartitioner]]
183+
* the default org.apache.spark.HashPartitioner
184184
* and [[org.apache.spark.bagel.DefaultCombiner]]
185185
*/
186186
def run[K: Manifest, V <: Vertex : Manifest, M <: Message[K] : Manifest](

‎core/pom.xml

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -225,7 +225,7 @@
225225
</goals>
226226
<configuration>
227227
<exportAntProperties>true</exportAntProperties>
228-
<tasks>
228+
<target>
229229
<property name="spark.classpath" refid="maven.test.classpath" />
230230
<property environment="env" />
231231
<fail message="Please set the SCALA_HOME (or SCALA_LIBRARY_PATH if scala is on the path) environment variables and retry.">
@@ -238,7 +238,7 @@
238238
</not>
239239
</condition>
240240
</fail>
241-
</tasks>
241+
</target>
242242
</configuration>
243243
</execution>
244244
</executions>

‎core/src/main/scala/org/apache/spark/SparkContext.scala

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -351,7 +351,7 @@ class SparkContext(
351351
* using the older MapReduce API (`org.apache.hadoop.mapred`).
352352
*
353353
* @param conf JobConf for setting up the dataset
354-
* @param inputFormatClass Class of the [[InputFormat]]
354+
* @param inputFormatClass Class of the InputFormat
355355
* @param keyClass Class of the keys
356356
* @param valueClass Class of the values
357357
* @param minSplits Minimum number of Hadoop Splits to generate.

‎core/src/main/scala/org/apache/spark/scheduler/JobLogger.scala

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -81,7 +81,7 @@ class JobLogger(val user: String, val logDirName: String)
8181
/**
8282
* Create a log file for one job
8383
* @param jobID ID of the job
84-
* @exception FileNotFoundException Fail to create log file
84+
* @throws FileNotFoundException Fail to create log file
8585
*/
8686
protected def createLogWriter(jobID: Int) {
8787
try {

‎core/src/main/scala/org/apache/spark/util/IndestructibleActorSystem.scala

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -23,9 +23,9 @@ import scala.util.control.{ControlThrowable, NonFatal}
2323
import com.typesafe.config.Config
2424

2525
/**
26-
* An [[akka.actor.ActorSystem]] which refuses to shut down in the event of a fatal exception.
26+
* An akka.actor.ActorSystem which refuses to shut down in the event of a fatal exception
2727
* This is necessary as Spark Executors are allowed to recover from fatal exceptions
28-
* (see [[org.apache.spark.executor.Executor]]).
28+
* (see org.apache.spark.executor.Executor)
2929
*/
3030
object IndestructibleActorSystem {
3131
def apply(name: String, config: Config): ActorSystem =

‎core/src/main/scala/org/apache/spark/util/StatCounter.scala

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -19,9 +19,9 @@ package org.apache.spark.util
1919

2020
/**
2121
* A class for tracking the statistics of a set of numbers (count, mean and variance) in a
22-
* numerically robust way. Includes support for merging two StatCounters. Based on
23-
* [[http://en.wikipedia.org/wiki/Algorithms_for_calculating_variance
24-
* Welford and Chan's algorithms for running variance]].
22+
* numerically robust way. Includes support for merging two StatCounters. Based on Welford
23+
* and Chan's [[http://en.wikipedia.org/wiki/Algorithms_for_calculating_variance algorithms]]
24+
* for running variance.
2525
*
2626
* @constructor Initialize the StatCounter with the given values.
2727
*/

‎core/src/main/scala/org/apache/spark/util/Vector.scala

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -136,7 +136,7 @@ object Vector {
136136

137137
/**
138138
* Creates this [[org.apache.spark.util.Vector]] of given length containing random numbers
139-
* between 0.0 and 1.0. Optional [[scala.util.Random]] number generator can be provided.
139+
* between 0.0 and 1.0. Optional scala.util.Random number generator can be provided.
140140
*/
141141
def random(length: Int, random: Random = new XORShiftRandom()) =
142142
Vector(length, _ => random.nextDouble())

‎core/src/test/java/org/apache/spark/JavaAPISuite.java

Lines changed: 25 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -75,8 +75,9 @@ public int compare(Integer a, Integer b) {
7575
else if (a < b) return 1;
7676
else return 0;
7777
}
78-
};
78+
}
7979

80+
@SuppressWarnings("unchecked")
8081
@Test
8182
public void sparkContextUnion() {
8283
// Union of non-specialized JavaRDDs
@@ -148,6 +149,7 @@ public void call(String s) {
148149
Assert.assertEquals(2, foreachCalls);
149150
}
150151

152+
@SuppressWarnings("unchecked")
151153
@Test
152154
public void lookup() {
153155
JavaPairRDD<String, String> categories = sc.parallelizePairs(Arrays.asList(
@@ -179,6 +181,7 @@ public Boolean call(Integer x) {
179181
Assert.assertEquals(5, oddsAndEvens.lookup(false).get(0).size()); // Odds
180182
}
181183

184+
@SuppressWarnings("unchecked")
182185
@Test
183186
public void cogroup() {
184187
JavaPairRDD<String, String> categories = sc.parallelizePairs(Arrays.asList(
@@ -197,6 +200,7 @@ public void cogroup() {
197200
cogrouped.collect();
198201
}
199202

203+
@SuppressWarnings("unchecked")
200204
@Test
201205
public void leftOuterJoin() {
202206
JavaPairRDD<Integer, Integer> rdd1 = sc.parallelizePairs(Arrays.asList(
@@ -243,6 +247,7 @@ public Integer call(Integer a, Integer b) {
243247
Assert.assertEquals(33, sum);
244248
}
245249

250+
@SuppressWarnings("unchecked")
246251
@Test
247252
public void foldByKey() {
248253
List<Tuple2<Integer, Integer>> pairs = Arrays.asList(
@@ -265,6 +270,7 @@ public Integer call(Integer a, Integer b) {
265270
Assert.assertEquals(3, sums.lookup(3).get(0).intValue());
266271
}
267272

273+
@SuppressWarnings("unchecked")
268274
@Test
269275
public void reduceByKey() {
270276
List<Tuple2<Integer, Integer>> pairs = Arrays.asList(
@@ -320,8 +326,8 @@ public void approximateResults() {
320326
public void take() {
321327
JavaRDD<Integer> rdd = sc.parallelize(Arrays.asList(1, 1, 2, 3, 5, 8, 13));
322328
Assert.assertEquals(1, rdd.first().intValue());
323-
List<Integer> firstTwo = rdd.take(2);
324-
List<Integer> sample = rdd.takeSample(false, 2, 42);
329+
rdd.take(2);
330+
rdd.takeSample(false, 2, 42);
325331
}
326332

327333
@Test
@@ -359,8 +365,8 @@ public Boolean call(Double x) {
359365
Assert.assertEquals(2.49444, rdd.stdev(), 0.01);
360366
Assert.assertEquals(2.73252, rdd.sampleStdev(), 0.01);
361367

362-
Double first = rdd.first();
363-
List<Double> take = rdd.take(5);
368+
rdd.first();
369+
rdd.take(5);
364370
}
365371

366372
@Test
@@ -438,11 +444,11 @@ public Iterable<Double> call(String s) {
438444
return lengths;
439445
}
440446
});
441-
Double x = doubles.first();
442-
Assert.assertEquals(5.0, doubles.first().doubleValue(), 0.01);
447+
Assert.assertEquals(5.0, doubles.first(), 0.01);
443448
Assert.assertEquals(11, pairs.count());
444449
}
445450

451+
@SuppressWarnings("unchecked")
446452
@Test
447453
public void mapsFromPairsToPairs() {
448454
List<Tuple2<Integer, String>> pairs = Arrays.asList(
@@ -509,6 +515,7 @@ public void repartition() {
509515
}
510516
}
511517

518+
@SuppressWarnings("unchecked")
512519
@Test
513520
public void persist() {
514521
JavaDoubleRDD doubleRDD = sc.parallelizeDoubles(Arrays.asList(1.0, 1.0, 2.0, 3.0, 5.0, 8.0));
@@ -573,6 +580,7 @@ public void textFilesCompressed() throws IOException {
573580
Assert.assertEquals(expected, readRDD.collect());
574581
}
575582

583+
@SuppressWarnings("unchecked")
576584
@Test
577585
public void sequenceFile() {
578586
File tempDir = Files.createTempDir();
@@ -602,6 +610,7 @@ public Tuple2<Integer, String> call(Tuple2<IntWritable, Text> pair) {
602610
Assert.assertEquals(pairs, readRDD.collect());
603611
}
604612

613+
@SuppressWarnings("unchecked")
605614
@Test
606615
public void writeWithNewAPIHadoopFile() {
607616
File tempDir = Files.createTempDir();
@@ -632,6 +641,7 @@ public String call(Tuple2<IntWritable, Text> x) {
632641
}).collect().toString());
633642
}
634643

644+
@SuppressWarnings("unchecked")
635645
@Test
636646
public void readWithNewAPIHadoopFile() throws IOException {
637647
File tempDir = Files.createTempDir();
@@ -674,6 +684,7 @@ public void objectFilesOfInts() {
674684
Assert.assertEquals(expected, readRDD.collect());
675685
}
676686

687+
@SuppressWarnings("unchecked")
677688
@Test
678689
public void objectFilesOfComplexTypes() {
679690
File tempDir = Files.createTempDir();
@@ -690,6 +701,7 @@ public void objectFilesOfComplexTypes() {
690701
Assert.assertEquals(pairs, readRDD.collect());
691702
}
692703

704+
@SuppressWarnings("unchecked")
693705
@Test
694706
public void hadoopFile() {
695707
File tempDir = Files.createTempDir();
@@ -719,6 +731,7 @@ public String call(Tuple2<IntWritable, Text> x) {
719731
}).collect().toString());
720732
}
721733

734+
@SuppressWarnings("unchecked")
722735
@Test
723736
public void hadoopFileCompressed() {
724737
File tempDir = Files.createTempDir();
@@ -824,7 +837,7 @@ public Float zero(Float initialValue) {
824837
}
825838
};
826839

827-
final Accumulator<Float> floatAccum = sc.accumulator((Float) 10.0f, floatAccumulatorParam);
840+
final Accumulator<Float> floatAccum = sc.accumulator(10.0f, floatAccumulatorParam);
828841
rdd.foreach(new VoidFunction<Integer>() {
829842
public void call(Integer x) {
830843
floatAccum.add((float) x);
@@ -876,6 +889,7 @@ public void checkpointAndRestore() {
876889
Assert.assertEquals(Arrays.asList(1, 2, 3, 4, 5), recovered.collect());
877890
}
878891

892+
@SuppressWarnings("unchecked")
879893
@Test
880894
public void mapOnPairRDD() {
881895
JavaRDD<Integer> rdd1 = sc.parallelize(Arrays.asList(1,2,3,4));
@@ -900,6 +914,7 @@ public Tuple2<Integer, Integer> call(Tuple2<Integer, Integer> in) throws Excepti
900914

901915
}
902916

917+
@SuppressWarnings("unchecked")
903918
@Test
904919
public void collectPartitions() {
905920
JavaRDD<Integer> rdd1 = sc.parallelize(Arrays.asList(1, 2, 3, 4, 5, 6, 7), 3);
@@ -968,14 +983,14 @@ public void countApproxDistinctByKey() {
968983
@Test
969984
public void collectAsMapWithIntArrayValues() {
970985
// Regression test for SPARK-1040
971-
JavaRDD<Integer> rdd = sc.parallelize(Arrays.asList(new Integer[] { 1 }));
986+
JavaRDD<Integer> rdd = sc.parallelize(Arrays.asList(1));
972987
JavaPairRDD<Integer, int[]> pairRDD = rdd.map(new PairFunction<Integer, Integer, int[]>() {
973988
@Override
974989
public Tuple2<Integer, int[]> call(Integer x) throws Exception {
975990
return new Tuple2<Integer, int[]>(x, new int[] { x });
976991
}
977992
});
978993
pairRDD.collect(); // Works fine
979-
Map<Integer, int[]> map = pairRDD.collectAsMap(); // Used to crash with ClassCastException
994+
pairRDD.collectAsMap(); // Used to crash with ClassCastException
980995
}
981996
}

‎pom.xml

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -592,12 +592,13 @@
592592
<plugin>
593593
<groupId>org.apache.maven.plugins</groupId>
594594
<artifactId>maven-compiler-plugin</artifactId>
595-
<version>2.5.1</version>
595+
<version>3.1</version>
596596
<configuration>
597597
<source>${java.version}</source>
598598
<target>${java.version}</target>
599599
<encoding>UTF-8</encoding>
600600
<maxmem>1024m</maxmem>
601+
<fork>true</fork>
601602
</configuration>
602603
</plugin>
603604
<plugin>
@@ -612,7 +613,7 @@
612613
<plugin>
613614
<groupId>org.scalatest</groupId>
614615
<artifactId>scalatest-maven-plugin</artifactId>
615-
<version>1.0-M2</version>
616+
<version>1.0-RC2</version>
616617
<configuration>
617618
<reportsDirectory>${project.build.directory}/surefire-reports</reportsDirectory>
618619
<junitxml>.</junitxml>

‎repl/pom.xml

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -98,7 +98,7 @@
9898
</goals>
9999
<configuration>
100100
<exportAntProperties>true</exportAntProperties>
101-
<tasks>
101+
<target>
102102
<property name="spark.classpath" refid="maven.test.classpath" />
103103
<property environment="env" />
104104
<fail message="Please set the SCALA_HOME (or SCALA_LIBRARY_PATH if scala is on the path) environment variables and retry.">
@@ -111,7 +111,7 @@
111111
</not>
112112
</condition>
113113
</fail>
114-
</tasks>
114+
</target>
115115
</configuration>
116116
</execution>
117117
</executions>

0 commit comments

Comments
 (0)