tweaked spark files

Author: sindbach

spark/Dockerfile

@@ -30,6 +30,7 @@ RUN wget -O mongo-hadoop-core-${MONGO_HADOOP_VERSION}.jar http://search.maven.or
 # get mongo java driver
 RUN wget http://central.maven.org/maven2/org/mongodb/mongo-java-driver/3.2.2/mongo-java-driver-${MONGO_JAVA_VERSION}.jar
 
+# Run single node of spark
 RUN ${HOME}/spark-${SPARK_VERSION}-bin-hadoop${HADOOP_VERSION}/sbin/start-master.sh
 
 COPY files/times.json /home/ubuntu/times.json

spark/files/reduceByKey.scala

@@ -26,6 +26,8 @@ mongoConfig.set("mongo.input.query", "{'myid':{$in:[1,2,3,4,5]}}")
 mongoConfig.set("mongo.input.sort", "{timestamp:-1}")
 
 val documents = sc.newAPIHadoopRDD(mongoConfig, classOf[MongoInputFormat], classOf[Object], classOf[BSONObject])
+// print out inputs
+documents.foreach(println)
 
 val outputRDD = documents.map(
             (tuple)=>((tuple._2.get("myid")), (tuple._2.get("timestamp")))
@@ -37,13 +39,12 @@ val outputRDD = documents.map(
                   }
         )
 
+// print out output
 outputRDD.foreach(println)
 
+// save result to MongoDB
 val outputConfig = new Configuration()
 outputConfig.set("mongo.output.uri", "mongodb://mongodb:27017/spark.output")
-
-// write out result
 outputRDD.saveAsNewAPIHadoopFile("file:///x", classOf[Any], classOf[Any], classOf[com.mongodb.hadoop.MongoOutputFormat[Any, Any]], outputConfig)
 
-
 System.exit(0);