M

webscraper/.gitignore

@@ -0,0 +1,13 @@
+# general things to ignore
+build/
+dist/
+*.egg-info/
+*.egg
+*.py[cod]
+__pycache__/
+*.so
+*~
+
+# due to using tox and pytest
+.tox
+.cache

webscraper/base.py

@@ -1,4 +1,5 @@
 from selenium.webdriver.support.ui import WebDriverWait
+from selenium.webdriver.support import expected_conditions as EC
 
 
 class Page(object):
@@ -13,6 +14,12 @@ def find_element(self, *locator):
     def find_elements(self, *locator):
         return self.driver.find_elements(*locator)
 
+    def staleness_of(self, *locator):
+        return EC.staleness_of(locator[1])
+
+    def presence_of(self, *locator):
+        return EC.presence_of_element_located(*locator)
+
     def get_title(self):
         return self.driver.title
 
@@ -23,4 +30,21 @@ def check_element_exists(self, *locator):
         return True if (self.find_element(*locator).is_selected()) else False
 
     def wait_for_element(self, *locator):
-        return WebDriverWait(self.driver, 30).until(lambda self: self.find_element(*locator))
+        return WebDriverWait(self.driver, 30).until(
+            lambda self: self.find_element(*locator))
+
+    def wait_for_element_clickable(self, *locator):
+        return WebDriverWait(self.driver, 30).until(
+            EC.element_to_be_clickable(*locator))
+
+    def wait_for_element_invisible(self, *locator):
+        return WebDriverWait(self.driver, 30).until(
+            EC.invisibility_of_element_located((locator)))
+
+    def wait_for_staleness(self, *locator):
+        return WebDriverWait(self.driver, 30).until(
+            self.staleness_of(*locator))
+
+    def presence_of_element_located(self, *locator):
+        return WebDriverWait(self.driver, 30).until(
+            self.presence_of(*locator))

webscraper/bet355_bet_history_scraper.py

@@ -1,8 +1,12 @@
 from selenium import webdriver
 from pages import *
+import json
+import io
 
 
 def main():
+    bets = []
+
     driver = webdriver.Firefox(executable_path='./geckodriver')
     driver.get("https://www.bet365.com/?lng=1&cb=10326429708#/HO/")
 
@@ -12,8 +16,19 @@ def main():
     main_page = MainPage(driver)
     main_page.login()
 
+    driver.get("https://members.bet365.com/MEMBERS/History/SportsHistory/HistorySearch/?BetStatus=0&SearchScope=2&platform=Desktop")
+
     popup_window = PopupWindow(driver)
-    popup_window.get_bet_history()
+
+    bets = popup_window.get_bet_history()
+
+    with io.open('bets.json', 'w', encoding="utf-8") as f:
+        f.write(unicode(json.dumps(bets, ensure_ascii=False)))
+
+    with io.open('bets.json', encoding="utf-8") as f:
+        data = json.load(f)
+        for ind, b in enumerate(data):
+            print("bet #" + str(ind + 1) + " :  " + b['bet_result'])
 
 
 if __name__ == '__main__':

webscraper/bets.json

@@ -0,0 +1 @@
+[{"bet_result": "Lost", "bet_event": "Sevilla v Villarreal\n(First Half Goals)", "bet_date": "05/02/2017", "bet_odds": "6/1", "bet_return": "0.00", "bet_id": "Bet Confirmation - AF5506054910F - Flash", "bet_type": "Over 0.5", "bet_stake": "2.00"}, {"bet_result": "Won", "bet_event": "Canberra Utd Women v Melbourne City Women\n(Match Goals)", "bet_date": "05/02/2017", "bet_odds": "8/11", "bet_return": "17.11", "bet_id": "Bet Confirmation - XF9734077110W - Mobile", "bet_type": "Under 1.5", "bet_stake": "10.00"}, {"bet_result": "Won", "bet_event": "CA River Plate v Lanus\n(Match Goals)", "bet_date": "05/02/2017", "bet_odds": "1/1", "bet_return": "20.00", "bet_id": "Bet Confirmation - JF4650080210F - Flash", "bet_type": "Over 0.5", "bet_stake": "10.00"}, {"bet_result": "Won", "bet_event": "Ceara v Ferroviario\n(Match Goals)", "bet_date": "05/02/2017", "bet_odds": "1/1", "bet_return": "20.00", "bet_id": "Bet Confirmation - YF3027964010F - Flash", "bet_type": "Over 0.5", "bet_stake": "10.00"}, {"bet_result": "Lost", "bet_event": "CD River Plate Ecuador v Emelec\n(Match Goals)", "bet_date": "04/02/2017", "bet_odds": "3/1", "bet_return": "0.00", "bet_id": "Bet Confirmation - LF6632670610F - Flash", "bet_type": "Over 0.5", "bet_stake": "5.00"}, {"bet_result": "Lost", "bet_event": "EC Bahia v Moto Club de São Luís\n(Alternative Total Corners)", "bet_date": "04/02/2017", "bet_odds": "4/9", "bet_return": "0.00", "bet_id": "Bet Confirmation - EF9478330310F - Flash", "bet_type": "Over 11.0", "bet_stake": "10.00"}, {"bet_result": "Lost", "bet_event": "Caldense MG v Villa Nova\n(Total Corners)", "bet_date": "04/02/2017", "bet_odds": "13/8", "bet_return": "0.00", "bet_id": "Bet Confirmation - HX7389082410W - Mobile", "bet_type": "Over 13.0", "bet_stake": "5.00"}, {"bet_result": "Won", "bet_event": "Caldense MG v Villa Nova\n(Match Goals)", "bet_date": "04/02/2017", "bet_odds": "11/8", "bet_return": "23.75", "bet_id": "Bet Confirmation - AX1495965410W - Mobile", "bet_type": "Over 0.5", "bet_stake": "10.00"}, {"bet_result": "Lost", "bet_event": "FC Porto v Sporting\n(Double Chance)", "bet_date": "04/02/2017", "bet_odds": "4/1", "bet_return": "0.00", "bet_id": "Bet Confirmation - QF9394712610W - Mobile", "bet_type": "Sporting or Draw", "bet_stake": "0.99"}, {"bet_result": "Lost", "bet_event": "Casertana v Lecce\n(Double Chance)", "bet_date": "04/02/2017", "bet_odds": "4/6", "bet_return": "0.00", "bet_id": "Bet Confirmation - XF1865548010F - Flash", "bet_type": "Lecce or Draw", "bet_stake": "10.00"}]