Merge pull request jleetutorial#5 from jleetutorial/pedro-changes_for_videos

Pedro changes for videos

Author: jleetutorial

build.gradle

@@ -1,9 +1,8 @@
-from pyspark import SparkContext
+from pyspark import SparkContext, SparkConf
 
 if __name__ == "__main__":
-
-    sc = SparkContext("local", "wordCounts")
-    sc.setLogLevel("ERROR")
+    conf = SparkConf().setAppName("wordCounts").setMaster("local[3]")
+    sc = SparkContext(conf = conf)
 
     lines = sc.textFile("in/word_count.text")
     wordRdd = lines.flatMap(lambda line: line.split(" "))

gradle/wrapper/gradle-wrapper.jar

@@ -1,9 +1,8 @@
-from pyspark import SparkContext
+from pyspark import SparkContext, SparkConf
 
 if __name__ == "__main__":
-
-    sc = SparkContext("local", "create")
-    sc.setLogLevel("ERROR")
+    conf = SparkConf().setAppName("create").setMaster("local")
+    sc = SparkContext(conf = conf)
 
     inputStrings = ["Lily 23", "Jack 29", "Mary 29", "James 8"]
     regularRDDs = sc.parallelize(inputStrings)

gradle/wrapper/gradle-wrapper.properties

@@ -1,9 +1,8 @@
-from pyspark import SparkContext
+from pyspark import SparkContext, SparkConf
 
 if __name__ == "__main__":
-
-    sc = SparkContext("local", "create")
-    sc.setLogLevel("ERROR")
+    conf = SparkConf().setAppName("create").setMaster("local")
+    sc = SparkContext(conf = conf)
 
     tuples = [("Lily", 23), ("Jack", 29), ("Mary", 29), ("James", 8)]
     pairRDD = sc.parallelize(tuples)

gradlew

@@ -1,18 +1,21 @@
-from pyspark import SparkContext
+from pyspark import SparkContext, SparkConf
 
 if __name__ == "__main__":
-
-    sc = SparkContext("local", "GroupByKeyVsReduceByKey")
-    sc.setLogLevel("ERROR")
+    conf = SparkConf().setAppName('GroupByKeyVsReduceByKey').setMaster("local[*]") 
+    sc = SparkContext(conf = conf)
 
     words = ["one", "two", "two", "three", "three", "three"]
     wordsPairRdd = sc.parallelize(words).map(lambda word: (word, 1))
 
-    wordCountsWithReduceByKey = wordsPairRdd.reduceByKey(lambda x, y: x + y).collect()
+    wordCountsWithReduceByKey = wordsPairRdd \
+        .reduceByKey(lambda x, y: x + y) \
+        .collect()
     print("wordCountsWithReduceByKey: {}".format(list(wordCountsWithReduceByKey)))
 
     wordCountsWithGroupByKey = wordsPairRdd \
         .groupByKey() \
-        .mapValues(lambda intIterable: len(intIterable)) \
+        .mapValues(len) \
         .collect()
     print("wordCountsWithGroupByKey: {}".format(list(wordCountsWithGroupByKey)))
+
+

gradlew.bat

@@ -1,9 +1,8 @@
-from pyspark import SparkContext
+from pyspark import SparkContext, SparkConf
 
 if __name__ == "__main__":
-
-    sc = SparkContext("local", "JoinOperations")
-    sc.setLogLevel("ERROR")
+    conf = SparkConf().setAppName("JoinOperations").setMaster("local[1]")
+    sc = SparkContext(conf = conf)
 
     ages = sc.parallelize([("Tom", 29), ("John", 22)])
     addresses = sc.parallelize([("James", "USA"), ("John", "UK")])