union example

Author: sparkcodegeeks

src/main/scala/com/sparkbyexamples/spark/dataframe/UnionExample.scala

@@ -0,0 +1,42 @@
+package com.sparkbyexamples.spark.dataframe
+
+import org.apache.spark.sql.SparkSession
+
+object UnionExample extends App{
+
+  val spark: SparkSession = SparkSession.builder()
+    .master("local[1]")
+    .appName("SparkByExamples.com")
+    .getOrCreate()
+
+  spark.sparkContext.setLogLevel("ERROR")
+
+  import spark.implicits._
+
+  val simpleData = Seq(("James","Sales","NY",90000,34,10000),
+    ("Michael","Sales","NY",86000,56,20000),
+    ("Robert","Sales","CA",81000,30,23000),
+    ("Maria","Finance","CA",90000,24,23000)
+  )
+  val df = simpleData.toDF("employee_name","department","state","salary","age","bonus")
+  df.printSchema()
+  df.show()
+
+  val simpleData2 = Seq(("James","Sales","NY",90000,34,10000),
+    ("Maria","Finance","CA",90000,24,23000),
+    ("Jen","Finance","NY",79000,53,15000),
+    ("Jeff","Marketing","CA",80000,25,18000),
+    ("Kumar","Marketing","NY",91000,50,21000)
+  )
+  val df2 = simpleData2.toDF("employee_name","department","state","salary","age","bonus")
+  df2.show(false)
+
+  val df3 = df.union(df2)
+  df3.show(false)
+  df3.distinct().show(false)
+
+  val df4 = df.unionAll(df2)
+  df4.show(false)
+
+
+}

src/main/scala/com/sparkbyexamples/spark/dataframe/WhereExample.scala

@@ -37,23 +37,23 @@ object WhereExample extends App{
   df.show()
 
   //Condition
-  df.filter(df("state") === "OH")
+  df.where(df("state") === "OH")
     .show(false)
 
   //SQL Expression
-  df.filter("gender == 'M'")
+  df.where("gender == 'M'")
     .show(false)
 
   //multiple condition
-  df.filter(df("state") === "OH" && df("gender") === "M")
+  df.where(df("state") === "OH" && df("gender") === "M")
     .show(false)
 
   //Array condition
-  df.filter(array_contains(df("languages"),"Java"))
+  df.where(array_contains(df("languages"),"Java"))
     .show(false)
 
   //Struct condition
-  df.filter(df("name.lastname") === "Williams")
+  df.where(df("name.lastname") === "Williams")
     .show(false)
 
 }