@@ -594,7 +594,8 @@ def test_oldhadoop(self):
594
594
"mapred.output.format.class" : "org.apache.hadoop.mapred.SequenceFileOutputFormat" ,
595
595
"mapred.output.key.class" : "org.apache.hadoop.io.IntWritable" ,
596
596
"mapred.output.value.class" : "org.apache.hadoop.io.MapWritable" ,
597
- "mapred.output.dir" : basepath + "/olddataset/" }
597
+ "mapred.output.dir" : basepath + "/olddataset/"
598
+ }
598
599
self .sc .parallelize (dict_data ).saveAsHadoopDataset (conf )
599
600
input_conf = {"mapred.input.dir" : basepath + "/olddataset/" }
600
601
old_dataset = sorted (self .sc .hadoopRDD (
@@ -624,11 +625,13 @@ def test_newhadoop(self):
624
625
valueConverter = "org.apache.spark.api.python.WritableToDoubleArrayConverter" ).collect ())
625
626
self .assertEqual (result , array_data )
626
627
627
- conf = {"mapreduce.outputformat.class" :
628
+ conf = {
629
+ "mapreduce.outputformat.class" :
628
630
"org.apache.hadoop.mapreduce.lib.output.SequenceFileOutputFormat" ,
629
- "mapred.output.key.class" : "org.apache.hadoop.io.IntWritable" ,
630
- "mapred.output.value.class" : "org.apache.spark.api.python.DoubleArrayWritable" ,
631
- "mapred.output.dir" : basepath + "/newdataset/" }
631
+ "mapred.output.key.class" : "org.apache.hadoop.io.IntWritable" ,
632
+ "mapred.output.value.class" : "org.apache.spark.api.python.DoubleArrayWritable" ,
633
+ "mapred.output.dir" : basepath + "/newdataset/"
634
+ }
632
635
self .sc .parallelize (array_data ).saveAsNewAPIHadoopDataset (
633
636
conf ,
634
637
valueConverter = "org.apache.spark.api.python.DoubleArrayToWritableConverter" )
@@ -1012,8 +1015,7 @@ class NumPyTests(PySparkTestCase):
1012
1015
"""General PySpark tests that depend on numpy """
1013
1016
1014
1017
def test_statcounter_array (self ):
1015
- x = self .sc .parallelize (
1016
- [np .array ([1.0 , 1.0 ]), np .array ([2.0 , 2.0 ]), np .array ([3.0 , 3.0 ])])
1018
+ x = self .sc .parallelize ([np .array ([1.0 , 1.0 ]), np .array ([2.0 , 2.0 ]), np .array ([3.0 , 3.0 ])])
1017
1019
s = x .stats ()
1018
1020
self .assertSequenceEqual ([2.0 , 2.0 ], s .mean ().tolist ())
1019
1021
self .assertSequenceEqual ([1.0 , 1.0 ], s .min ().tolist ())
0 commit comments