Refactoring to cleaner code

Author: Miguel Yanez

scrapers/__init__.py renamed to flight_scraper/__init__.py

@@ -0,0 +1 @@
+__author__ = 'mayanez'

scrapers/engines/__init__.py renamed to flight_scraper/engines/__init__.py

@@ -0,0 +1,110 @@
+import json
+import logging
+import urllib
+from scrapers.solution_model import Seat, Flight, SeatQuery
+from selenium import webdriver
+
+logging.basicConfig(level=logging.INFO)
+
+class FlightStatsDriver(webdriver.PhantomJS):
+    __logger = logging.getLogger(__name__)
+    __base_url = "http://www.flightstats.com"
+    __request_uri = "/go/FlightAvailability/flightAvailability.do"
+    __http_header = {
+            'Host' : 'www.flightstats.com',
+            'Content-Type' : 'text/html'}
+    __params = {
+        'departure' : '',
+        'airline' : '',
+        'arrival' : '',
+        'connection' : '',
+        'queryDate' : '', #yyyy-mm-dd
+        'queryTime' : '2',
+        'excludeConnectionCodes' : '',
+        'cabinCode' : 'A',
+        'numOfSeats' : '1',
+        'queryType' : 'D',
+        'fareClassCodes' : ''}
+
+    @property
+    def origin(self):
+        return self.__origin
+
+    @origin.setter
+    def origin(self, origin):
+        self.__origin = origin
+
+    @property
+    def destination(self):
+        return self.__destination
+
+    @destination.setter
+    def destination(self, destination):
+        self.__destination = destination
+
+    @property
+    def depart_date(self):
+        return self.__depart_date
+
+    @depart_date.setter
+    def depart_date(self, depart_date):
+        self.__depart_date = depart_date
+
+    @property
+    def return_date(self):
+        return self.__return_date
+
+    @return_date.setter
+    def return_date(self, return_date):
+        self.__return_date = return_date
+
+    def __init__(self, executable_path, service_log_path):
+       webdriver.PhantomJS(executable_path=executable_path, service_log_path=service_log_path)
+
+    def __extract_flights_with_seats(self, json_obj):
+
+        flight_list = list()
+        self.__logger.info('Extracting flights with seats')
+        for k, results in json_obj.iteritems():
+            for k2, flights in results['flights'].iteritems():
+                airline = flights['airline']
+                fno = flights['flightNumber']
+                dep_city = flights['depCode']
+                arr_city = flights['arrCode']
+                flight = Flight(dep_city=dep_city, arr_city=arr_city, airline=airline, fno=fno, dep_time=self.depart_date)
+                seats = list()
+
+                for k3, cabin in flights['cabins'].iteritems():
+                    cabin_code = cabin['code']
+
+                    for fare_class, seat_availability in cabin['fares'].iteritems():
+                        if seat_availability == "":
+                            seat_availability = 0
+                        else:
+                            seat_availability = int(seat_availability)
+
+                        seat = Seat(cabin_code=cabin_code, fare_class=fare_class, availability=seat_availability)
+                        seats.append(seat)
+
+                flight.seats = seats
+                flight_list.append(flight)
+
+        return flight_list
+
+    def get_seat_availability(self):
+        params = urllib.urlencode(self.__params)
+        request_url = self.__base_url + self.__request_uri +("?%s" % params)
+        self.__logger.info('Requesting URL: %s' % (self.__request_url))
+        self.get(request_url)
+        self.__logger.info('Running Javascript to retrieve available routes')
+        result = self.execute_script('return JSON.stringify(availRoutes)')
+        j = json.loads(unicode(result))
+
+        flight_list = self.__extract_flights_with_seats(j)
+        self.__logger.info('Saving SeatQuery to Database')
+        seat_query = SeatQuery(flights=flight_list)
+        seat_query.save()
+        self.__logger.info('Quiting the Web Driver')
+        self.quit
+
+        return flight_list

flight_scraper/engines/flight_stats/__init__.py

@@ -4,7 +4,7 @@
 import logging
 
 from selenium import webdriver
-from scrapers.solution_model import *
+from flight_scraper.solution_model import *
 
 logging.basicConfig(level=logging.INFO)
 logger = logging.getLogger(__name__)

flight_scraper/engines/flight_stats/driver.py

@@ -0,0 +1 @@
+__author__ = 'mayanez'

scrapers/engines/flightstats.py renamed to flight_scraper/engines/flight_stats/flightstats.py

@@ -0,0 +1,113 @@
+import json
+import logging
+import datetime
+import requests
+from flight_scraper.solution_model import Solution, Flight, Itinerary
+
+logging.basicConfig(level=logging.INFO)
+
+class ItaMatrixDriver(object):
+    __logger = logging.getLogger(__name__)
+    engine = "ITA Matrix"
+    __base_url = "http://matrix.itasoftware.com"
+    __request_uri = "/xhr/shop/search?"
+    __http_header = {
+        'Host': 'matrix.itasoftware.com',
+        'Content-Type': 'application/x-www-form-urlencoded',
+        'Cache-Control': 'no-cache',
+        'Content-Length': '0'
+    }
+    __base_request = "name=specificDates&summarizers=carrierStopMatrix\
+%2CcurrencyNotice%2CsolutionList%2CitineraryPriceSlider%2C\
+itineraryCarrierList%2CitineraryDepartureTimeRanges%2CitineraryArrivalTimeRanges\
+%2CdurationSliderItinerary%2CitineraryOrigins%2CitineraryDestinations%2C\
+itineraryStopCountList%2CwarningsItinerary&format=JSON&inputs="
+    __json_request = json.loads('{"slices":[{"origins":["PDX"],"originPreferCity":false,"commandLine":"airlines AA DL AS UA","destinations":["SEA"],"destinationPreferCity":false,"date":"2013-06-07","isArrivalDate":false,"dateModifier":{"minus":0,"plus":0}},{"destinations":["PDX"],"destinationPreferCity":false,"origins":["SEA"],"originPreferCity":false,"commandLine":"airlines AA DL AS","date":"2013-06-09","isArrivalDate":false,"dateModifier":{"minus":0,"plus":0}}],"pax":{"adults":1},"cabin":"COACH","maxStopCount":0,"changeOfAirport":false,"checkAvailability":true,"page":{"size":2000},"sorts":"default"}')
+
+
+    @property
+    def origin(self):
+        return self.__json_request['slices'][0]['origins'][0]
+
+    @origin.setter
+    def origin(self, origin):
+        self.__json_request['slices'][0]['origins'][0] = origin
+        self.__json_request['slices'][1]['destinations'][0] = origin
+
+    @property
+    def destination(self):
+        return self.__json_request['slices'][0]['destinations'][0]
+
+    @destination.setter
+    def destination(self, destination):
+        self.__json_request['slices'][0]['destinations'][0] = destination
+        self.__json_request['slices'][1]['origins'][0] = destination
+
+    @property
+    def depart_date(self):
+        return datetime.datetime.strptime(self.__json_request['slices'][0]['date'], "%Y-%m-%d")
+
+    @depart_date.setter
+    def depart_date(self, depart_date):
+        self.__json_request['slices'][0]['date'] = depart_date.strftime('%Y-%m-%d')
+
+    @property
+    def return_date(self):
+        return datetime.datetime.strptime(self.__json_request['slices'][1]['date'], "%Y-%m-%d")
+
+    @return_date.setter
+    def return_date(self, return_date):
+        self.__json_request['slices'][1]['date'] = return_date.strftime('%Y-%m-%d')
+
+    def __init__(self, origin, destination, depart_date, return_date):
+        self.origin = origin
+        self.destination = destination
+        self.depart_date = depart_date
+        self.return_date = return_date
+
+    def build_solutions(self):
+        """
+            Builds search solution. Adds to MongoDB and returns the Solution object.
+        """
+        data = self.__base_request + json.dumps(self.__json_request)
+        request_url = self.__base_url + self.__request_uri + data
+
+        self.__logger.info('Making request to ITA Matrix: %s', (request_url))
+        print 'Making request to ITA Matrix: %s' % (request_url)
+        response = requests.post(request_url, headers=self.__http_header)
+        response_json = json.loads(response.text[4:])
+
+        print response_json
+        self.__logger.info('Creating objects to insert to database')
+        solution = Solution(engine=self.engine, origin=self.origin, destination=self.destination, depart_date=self.depart_date, return_date=self.return_date)
+        solution.min_price = response_json['result']['solutionList']['minPrice']
+
+        for sol in response_json['result']['solutionList']['solutions']:
+            origin_flight_airline = sol['itinerary']['slices'][0]['flights'][0][:2]
+            origin_flight_number = int(sol['itinerary']['slices'][0]['flights'][0][2:])
+            dep_time = datetime.datetime.strptime(sol['itinerary']['slices'][0]['departure'][:-6], "%Y-%m-%dT%H:%M")
+            arr_time = datetime.datetime.strptime(sol['itinerary']['slices'][0]['arrival'][:-6], "%Y-%m-%dT%H:%M")
+            arr_city = sol['itinerary']['slices'][0]['destination']['code']
+            dep_city = sol['itinerary']['slices'][0]['origin']['code']
+
+            origin_flight = Flight(airline=origin_flight_airline, fno=origin_flight_number, dep_city=dep_city, arr_city=arr_city, dep_time=dep_time, arr_time=arr_time)
+
+            return_flight_airline = sol['itinerary']['slices'][1]['flights'][0][:2]
+            return_flight_number = int(sol['itinerary']['slices'][1]['flights'][0][2:])
+            dep_time = datetime.datetime.strptime(sol['itinerary']['slices'][1]['departure'][:-6], "%Y-%m-%dT%H:%M")
+            arr_time = datetime.datetime.strptime(sol['itinerary']['slices'][1]['arrival'][:-6], "%Y-%m-%dT%H:%M")
+            arr_city = sol['itinerary']['slices'][1]['destination']['code']
+            dep_city = sol['itinerary']['slices'][1]['origin']['code']
+
+            return_flight = Flight(airline=return_flight_airline, fno=return_flight_number, dep_city=dep_city, arr_city=arr_city, dep_time=dep_time, arr_time=arr_time)
+
+            flight_list = [origin_flight, return_flight]
+            price = sol['displayTotal']
+            itinerary = Itinerary(flights=flight_list, price=price)
+            solution.itineraries.append(itinerary)
+
+        solution.save()
+
+        return solution
+
+

flight_scraper/engines/ita_matrix/__init__.py

@@ -0,0 +1,73 @@
+from scrapers.solution_model import Solution, SeatQuery
+from engines.ita_matrix.driver import ItaMatrixDriver
+
+
+class FlightScraper(object):
+
+    @property
+    def origin(self):
+        return self.__origin
+
+    @origin.setter
+    def origin(self, origin):
+        self.__origin = origin
+
+    @property
+    def destination(self):
+        return self.__destination
+
+    @destination.setter
+    def destination(self, destination):
+        self.__destination = destination
+
+    @property
+    def depart_date(self):
+        return self.__depart_date
+
+    @depart_date.setter
+    def depart_date(self, depart_date):
+        self.__depart_date = depart_date
+
+    @property
+    def return_date(self):
+        return self.__return_date
+
+    @return_date.setter
+    def return_date(self, return_date):
+        self.__return_date = return_date
+
+    def search_flights(self):
+        ita_driver = ItaMatrixDriver(origin=self.__origin, destination=self.__destination, depart_date=self.__depart_date, return_date=self.__return_date)
+        return ita_driver.build_solutions()
+
+    def solutions(self):
+        """
+        Returns a Solution object from MongoDB
+        """
+        return Solution.objects(depart_date=self.__depart_date, return_date=self.__return_date, origin=self.__origin, destination=self.__destination)
+
+    def itineraries(self, flights_to_match):
+        results = list()
+        solutions = self.get_solutions()
+
+        for sol in solutions:
+            itineraries = sol.itineraries
+            for itinerary in itineraries:
+                flights = set(itinerary.flights)
+                matched = flights.intersection(flights_to_match)
+                if len(matched) > 0:
+                    results.append(itinerary)
+
+        return results
+
+    def __get_seats(self, date):
+        seat_query = SeatQuery.objects(flights__dep_city=self.__origin, flights__arr_city=self.__destination, flights__dep_time=date)
+        return seat_query
+
+    def departure_seats(self):
+        return self.__get_seats(self.__depart_date)
+
+    def return_seats(self):
+        return self.__get_seats(self.__return_date)
+
+