pyspark examples

Author: sparkcodegeeks

pyspark-broadcast-dataframe.py

@@ -33,5 +33,5 @@ def state_convert(code):
 
 # Broadcast variable on filter
 
-filter_df= df.where((df['Foo'].isin(broadcastStates.value)))
+filteDf= df.where((df['state'].isin(broadcastStates.value)))
 

pyspark-withcolumn.py

@@ -2,34 +2,22 @@
 """
 Created on Sun Jun 14 10:20:19 2020
 """
-
 import pyspark
 from pyspark.sql import SparkSession
 from pyspark.sql.functions import col, lit
-from pyspark.sql.types import StructType, StructField, StringType
+from pyspark.sql.types import StructType, StructField, StringType,IntegerType
 
 spark = SparkSession.builder.appName('SparkByExamples.com').getOrCreate()
 
-data = [(("James","","Smith"),"36636","M","3000"), \
-      (("Michael","Rose",""),"40288","M","4000"), \
-      (("Robert","","Williams"),"42114","M","4000"), \
-      (("Maria","Anne","Jones"),"39192","F","4000"), \
-      (("Jen","Mary","Brown"),"","F","-1") \
+data = [('James','','Smith','1991-04-01','M',3000),
+  ('Michael','Rose','','2000-05-19','M',4000),
+  ('Robert','','Williams','1978-09-05','M',4000),
+  ('Maria','Anne','Jones','1967-12-01','F',4000),
+  ('Jen','Mary','Brown','1980-02-17','F',-1)
 ]
 
-schema = StructType([
-        StructField('name', StructType([
-             StructField('firstname', StringType(), True),
-             StructField('middlename', StringType(), True),
-             StructField('lastname', StringType(), True)
-             ])),
-          StructField('dob', StringType(), True),
-         StructField('gender', StringType(), True),
-         StructField('salary', StringType(), True)
-         ])
-
-
-df = spark.createDataFrame(data=data, schema = schema)
+columns = ["firstname","middlename","lastname","dob","gender","salary"]
+df = spark.createDataFrame(data=data, schema = columns)
 df.printSchema()
 df.show(truncate=False)
 
@@ -51,13 +39,36 @@
    .withColumn("anotherColumn",lit("anotherValue"))
 df6.printSchema()
 
-
 df.withColumnRenamed("gender","sex") \
   .show(truncate=False) 
 
 df4.drop("CopiedColumn") \
 .show(truncate=False) 
 
+dataStruct = [(("James","","Smith"),"36636","M","3000"), \
+      (("Michael","Rose",""),"40288","M","4000"), \
+      (("Robert","","Williams"),"42114","M","4000"), \
+      (("Maria","Anne","Jones"),"39192","F","4000"), \
+      (("Jen","Mary","Brown"),"","F","-1") \
+]
+
+schemaStruct = StructType([
+        StructField('name', StructType([
+             StructField('firstname', StringType(), True),
+             StructField('middlename', StringType(), True),
+             StructField('lastname', StringType(), True)
+             ])),
+          StructField('dob', StringType(), True),
+         StructField('gender', StringType(), True),
+         StructField('salary', StringType(), True)
+         ])
+
+
+df7 = spark.createDataFrame(data=dataStruct, schema = schemaStruct)
+df7.printSchema()
+df7.show(truncate=False)
+
+
 """
 columns = ["name","address"]
 data = [("Robert, Smith", "1 Main st, Newark, NJ, 92537"), \