Handling Null values

Author: sparkcodegeeks

src/main/scala/com/sparkbyexamples/spark/dataframe/FilterExample.scala

@@ -39,15 +39,15 @@ object FilterExample extends App{
   df.filter(df("state") === "OH")
     .show(false)
 
-  //Expression
+  //SQL Expression
   df.filter("gender == 'M'")
     .show(false)
 
   //multiple condition
   df.filter(df("state") === "OH" && df("gender") === "M")
     .show(false)
 
-  //Row condition
+  //Array condition
   df.filter(array_contains(df("languages"),"Java"))
     .show(false)
 

src/main/scala/com/sparkbyexamples/spark/dataframe/HandleNullExample.scala

@@ -0,0 +1,36 @@
+package com.sparkbyexamples.spark.dataframe
+
+import org.apache.spark.sql.SparkSession
+
+object HandleNullExample extends App{
+
+  val spark: SparkSession = SparkSession.builder()
+    .master("local[1]")
+    .appName("SparkByExamples.com")
+    .getOrCreate()
+
+  val filePath="src/main/resources/small_zipcode.csv"
+
+  val df = spark.read.options(Map("inferSchema"->"true","delimiter"->",","header"->"true")).csv(filePath)
+  df.printSchema()
+  df.show(false)
+
+  df.na.fill(0)
+    .show(false)
+
+  df.na.fill(0,Array("population"))
+    .show(false)
+
+  df.na.fill("")
+    .show(false)
+
+  df.na.fill("unknown",Array("city"))
+    .na.fill("",Array("type"))
+    .show(false)
+
+
+
+  // Array and map columns
+
+
+}

src/main/scala/com/sparkbyexamples/spark/dataframe/WhereExample.scala

@@ -0,0 +1,59 @@
+package com.sparkbyexamples.spark.dataframe
+
+import org.apache.spark.sql.functions.array_contains
+import org.apache.spark.sql.types.{ArrayType, StringType, StructType}
+import org.apache.spark.sql.{Row, SparkSession}
+
+object WhereExample extends App{
+
+  val spark: SparkSession = SparkSession.builder()
+    .master("local[1]")
+    .appName("SparkByExamples.com")
+    .getOrCreate()
+
+  spark.sparkContext.setLogLevel("ERROR")
+
+  val arrayStructureData = Seq(
+    Row(Row("James","","Smith"),List("Java","Scala","C++"),"OH","M"),
+    Row(Row("Anna","Rose",""),List("Spark","Java","C++"),"NY","F"),
+    Row(Row("Julia","","Williams"),List("CSharp","VB"),"OH","F"),
+    Row(Row("Maria","Anne","Jones"),List("CSharp","VB"),"NY","M"),
+    Row(Row("Jen","Mary","Brown"),List("CSharp","VB"),"NY","M"),
+    Row(Row("Mike","Mary","Williams"),List("Python","VB"),"OH","M")
+  )
+
+  val arrayStructureSchema = new StructType()
+    .add("name",new StructType()
+      .add("firstname",StringType)
+      .add("middlename",StringType)
+      .add("lastname",StringType))
+    .add("languages", ArrayType(StringType))
+    .add("state", StringType)
+    .add("gender", StringType)
+
+  val df = spark.createDataFrame(
+    spark.sparkContext.parallelize(arrayStructureData),arrayStructureSchema)
+  df.printSchema()
+  df.show()
+
+  //Condition
+  df.filter(df("state") === "OH")
+    .show(false)
+
+  //SQL Expression
+  df.filter("gender == 'M'")
+    .show(false)
+
+  //multiple condition
+  df.filter(df("state") === "OH" && df("gender") === "M")
+    .show(false)
+
+  //Array condition
+  df.filter(array_contains(df("languages"),"Java"))
+    .show(false)
+
+  //Struct condition
+  df.filter(df("name.lastname") === "Williams")
+    .show(false)
+
+}