Bugfix: Added up to 10 retries for defeating cloudflare protection

__init__.py

@@ -4,7 +4,7 @@
 class KoboMetadata(Source):
     name = "Kobo Metadata"
     author = "NotSimone"
-    version = (1, 6, 2)
+    version = (1, 6, 3)
     minimum_calibre_version = (5, 0, 0)
     description = _("Downloads metadata and covers from Kobo")
 

kobo_metadata.py

@@ -96,10 +96,18 @@ def _get_session(self) -> requests.Session:
     def _get_webpage(self, url: str, timeout: int, log: Log) -> Tuple[Optional[html.Element], bool]:
         session = self._get_session()
         try:
-            resp = session.get(url, timeout=timeout)
-            page = html.fromstring(resp.text)
-            is_search = "/search?" in resp.url
-            return (page, is_search)
+            attempts = 0
+            while attempts < 10:
+                resp = session.get(url, timeout=timeout)
+                page = html.fromstring(resp.text)
+                # If we failed to get past the cloudflare protection, we get a page with this class
+                if not page.xpath("//form[@class='challenge-form']"):
+                    is_search = "/search?" in resp.url
+                    return (page, is_search)
+                log.info(f"KoboMetadata::get_webpage: Could not defeat cloudflare protection - trying again for {url}")
+                attempts += 1
+            log.error(f"KoboMetadata::get_webpage: Could not defeat cloudflare protection - giving up for {url}")
+            return (None, False)
         except Exception as e:
             log.error(f"KoboMetadata::get_webpage: Got exception while opening url: {e}")
             return (None, False)