Airport Problem 1

Author: bhasharm7

PlayingWithSpark/.idea/.name

@@ -0,0 +1,30 @@
+package rdd.airports
+
+import utils.{Context, Utils}
+
+
+/* Create a Spark program to read the airport data from in/airports.text,
+          find all the airports whose latitude are bigger than 40.
+          Then output the airport's name and the airport's latitude to out/airports_by_latitude.text.
+          Each row of the input file contains the following columns:
+          Airport ID, Name of airport, Main city served by airport, Country where airport is located, IATA/FAA code,
+          ICAO Code, Latitude, Longitude, Altitude, Timezone, DST, Timezone in Olson format
+          Sample output:
+          "St Anthony", 51.391944
+          "Tofino", 49.082222
+          ...
+        */
+object AirportsByLatitudeProblem extends App with Context {
+
+  val airports = sparkSession.sparkContext.textFile("in/airports.text")
+
+  val airportLat = airports.filter(line => line.split(Utils.COMMA_DELIMITER)(6).toFloat > 40)
+
+  val airportLatAndName = airportLat.map(line => {
+    val split = line.split(Utils.COMMA_DELIMITER)
+    split(1) + " , " + split(6)
+  })
+
+  airportLatAndName.saveAsTextFile("out/airportLatName.text")
+
+}

PlayingWithSpark/.idea/modules/PlayingWithSpark.iml

@@ -0,0 +1,32 @@
+package rdd.airports
+
+import org.apache.spark.{SparkConf, SparkContext}
+import utils.{Context, Utils}
+
+
+/* Create a Spark program to read the airport data from in/airports.text,
+           find all the airports which are located in United States
+           and output the airport's name and the city's name to out/airports_in_usa.text.
+           Each row of the input file contains the following columns:
+           Airport ID, Name of airport, Main city served by airport, Country where airport is located, IATA/FAA code,
+           ICAO Code, Latitude, Longitude, Altitude, Timezone, DST, Timezone in Olson format
+           Sample output:
+           "Putnam County Airport", "Greencastle"
+           "Dowagiac Municipal Airport", "Dowagiac"
+           ...
+         */
+object AirportsInUSA extends App with Context {
+
+  val airports = sparkSession.sparkContext.textFile("in/airports.text")
+
+  val airportsInUSA = airports.filter(lines => lines.split(Utils.COMMA_DELIMITER)(3) == "\"United States\"")
+
+  airportsInUSA.foreach(println)
+
+  val airportNameAndCity = airportsInUSA.map(line => {
+    val splits = line.split(Utils.COMMA_DELIMITER)
+    splits(1) + "," + splits(2)
+  })
+
+  airportNameAndCity.saveAsTextFile("out/airport_in_usa.txt")
+}

PlayingWithSpark/.idea/sbt.xml

@@ -1,10 +1,11 @@
 package utils
 
-import org.apache.spark.SparkConf
 import org.apache.spark.sql.SparkSession
 
 trait Context {
 
+  System.setProperty("hadoop.home.dir", "C:/hadoop")
+
   lazy val sparkSession: SparkSession = SparkSession
     .builder()
     .appName("PlayingWithSpark")

PlayingWithSpark/.idea/workspace.xml

@@ -0,0 +1,5 @@
+package utils
+
+object Utils {
+  val COMMA_DELIMITER = ",(?=([^\"]*\"[^\"]*\")*[^\"]*$)"
+}

PlayingWithSpark/project/target/config-classes/$4bd11ea989eb68f0429e.cache

@@ -4,3 +4,5 @@ exit
 exit
 ;set _root_.scala.collection.Seq(shellPrompt := { _ => "" },SettingKey[_root_.scala.Option[_root_.sbt.File]]("sbtStructureOutputFile") in _root_.sbt.Global := _root_.scala.Some(_root_.sbt.file("C:/Users/bhasharm7/AppData/Local/Temp/sbt-structure1.xml")),SettingKey[_root_.java.lang.String]("sbtStructureOptions") in _root_.sbt.Global := "download, resolveClassifiers");apply -cp "C:/Users/bhasharm7/.IdeaIC2018.2/config/plugins/Scala/launcher/sbt-structure-1.0.jar" org.jetbrains.sbt.CreateTasks;*/*:dumpStructure
 exit
+;set _root_.scala.collection.Seq(shellPrompt := { _ => "" },SettingKey[_root_.scala.Option[_root_.sbt.File]]("sbtStructureOutputFile") in _root_.sbt.Global := _root_.scala.Some(_root_.sbt.file("C:/Users/bhasharm7/AppData/Local/Temp/sbt-structure.xml")),SettingKey[_root_.java.lang.String]("sbtStructureOptions") in _root_.sbt.Global := "download, resolveClassifiers");apply -cp "C:/Users/bhasharm7/.IdeaIC2018.2/config/plugins/Scala/launcher/sbt-structure-1.0.jar" org.jetbrains.sbt.CreateTasks;*/*:dumpStructure
+exit

PlayingWithSpark/project/target/streams/$global/update/$global/streams/update_cache_2.12/output

@@ -1,5 +1,5 @@
 #default#playingwithspark$sources;0.1 resolved revisions
-#Tue Jun 04 11:58:24 IST 2019
+#Tue Jun 04 13:00:33 IST 2019
 +organisation\:\#@\#\:+org.slf4j\:\#@\#\:+branch\:\#@\#\:+@\#\:NULL\:\#@\:\#@\#\:+module\:\#@\#\:+slf4j-api\:\#@\#\:+revision\:\#@\#\:+1.7.16\:\#@\#\:=1.7.16 release 1.7.16 null
 +organisation\:\#@\#\:+org.xerial.snappy\:\#@\#\:+branch\:\#@\#\:+@\#\:NULL\:\#@\:\#@\#\:+module\:\#@\#\:+snappy-java\:\#@\#\:+revision\:\#@\#\:+1.1.2.6\:\#@\#\:=1.1.2.6 release 1.1.2.6 null
 +organisation\:\#@\#\:+org.apache.hadoop\:\#@\#\:+branch\:\#@\#\:+@\#\:NULL\:\#@\:\#@\#\:+module\:\#@\#\:+hadoop-common\:\#@\#\:+revision\:\#@\#\:+2.6.5\:\#@\#\:=2.6.5 release 2.6.5 null

PlayingWithSpark/src/main/scala/rdd/airports/AirportsByLatitudeProblem.scala

@@ -4,7 +4,7 @@
 		module="playingwithspark$sources"
 		revision="0.1"
 		status="integration"
-		publication="20190604115814"
+		publication="20190604125951"
 	/>
 	<configurations>
 		<conf name="compile" visibility="public" description=""/>