Implement retry mechanism for the crawler

wakealert/constants/__init__.py

@@ -1,6 +1,7 @@
 import os
 
 POLLING_INTERVAL = 120.0
+TIMEOUT_GET_REQUEST = 10.0
 CANCELLED = "CANCELLED"
 FREE = "FREE"
 EMPTY = "EMPTY"

wakealert/helpers/run.py

@@ -2,7 +2,7 @@
 from time import sleep
 from zoneinfo import ZoneInfo
 
-from ..constants import POLLING_INTERVAL
+from ..constants import POLLING_INTERVAL, RUN_WITH_LOGS
 from ..helpers.web import (
     create_browser_session,
     get_reservations_html,
@@ -15,11 +15,26 @@
 from ..helpers.telegram_bot import notify_to_telegram
 
 
-def run_once(email, password, token, chat_id, session=None):
+def run_once(email, password, token, chat_id, session=None, retries=0):
+    if retries > 0:
+        if RUN_WITH_LOGS:
+            print(f"Sleep {2**retries} seconds before retrying")
+        sleep(2**retries)
     browser_session = create_browser_session() if session is None else session
-    html = get_reservations_html(
-        session=browser_session, email=email, password=password
-    )
+    try:
+        html = get_reservations_html(
+            session=browser_session, email=email, password=password
+        )
+    except Exception:
+        # Crete new browser session when retrying to avoid RemoteDisconnected errors
+        return run_once(
+            email=email,
+            password=password,
+            token=token,
+            chat_id=chat_id,
+            session=None,
+            retries=retries + 1,
+        )
 
     calendar_entries = extract_calendar_entries(html)
     slots = find_available_slots(calendar_entries)

wakealert/helpers/web.py

@@ -2,7 +2,7 @@
 
 from requests import Session
 
-from ..constants import RUN_WITH_LOGS
+from ..constants import RUN_WITH_LOGS, TIMEOUT_GET_REQUEST
 
 HOME_PAGE = "https://lakeridersclub.ch/index.php"
 AUTHENTICATION_PAGE = "https://lakeridersclub.ch/membres/connexion.php"
@@ -41,11 +41,11 @@ def _print_crawling_log():
 
 
 def get_reservations_html(session, email, password):
-    response = session.get(CALENDAR_PAGE)
+    response = session.get(CALENDAR_PAGE, timeout=TIMEOUT_GET_REQUEST)
     if response.url == HOME_PAGE:
         # The session is not authenticated. Re-authenticate
         get_session_authorized(session=session, email=email, password=password)
-        authenticated_response = session.get(CALENDAR_PAGE)
+        authenticated_response = session.get(CALENDAR_PAGE, timeout=TIMEOUT_GET_REQUEST)
         if authenticated_response.url == HOME_PAGE:
             raise Exception("Invalid credentials")
         _print_crawling_log()