refactor and test in dockerfile

.devcontainer/devcontainer.json

@@ -24,7 +24,6 @@
       }
     }
   },
-  "postStartCommand": "./entrypoint.sh",
   // Features to add to the dev container. More info: https://containers.dev/features.
   // "features": {},
   // Use 'forwardPorts' to make a list of ports inside the container available locally.

Dockerfile

@@ -55,4 +55,5 @@ RUN poetry install
 COPY fix_nodriver.py ./
 RUN . /app/.venv/bin/activate && python fix_nodriver.py
 COPY . .
+RUN ./run_vnc.sh && . /app/.venv/bin/activate && poetry run pytest -n auto
 CMD ["./entrypoint.sh"]

entrypoint.sh

@@ -1,19 +1,6 @@
 #!/bin/sh
 
-rm -f /tmp/.X0-lock
-
-# Run Xvfb on dispaly 0.
-Xvfb :0 -screen 0 1280x720x16 &
-
-# Run fluxbox windows manager on display 0.
-fluxbox -display :0 &
-
-# Run x11vnc on display 0
-x11vnc -display :0 -forever -ncache 10 &
-
-# Add delay
-sleep 5
+./run_vnc.sh
 
 # Activate virtual environment
-export DISPLAY=:0
 . .venv/bin/activate && python3 main.py

main.py

@@ -31,16 +31,18 @@ async def read_item(request: LinkRequest):
     logger.info(f"Request: {request}")
     start_time = int(time.time() * 1000)
     browser = await new_browser()
+    await asyncio.sleep(1)
     page = await browser.get(request.url)
     await page.bring_to_front()
     timeout = request.maxTimeout
     if timeout == 0:
         timeout = None
     try:
         challenged = await asyncio.wait_for(bypass_cloudflare(page), timeout=timeout)
+    except asyncio.TimeoutError:
+        logger.info("Timed out bypassing Cloudflare")
     except Exception as e:
-        logger.error(await page.get_content())
-        logger.fatal("Element is a string, please report this to Byparr dev")
+        browser.stop()
         raise HTTPException(detail="Couldn't bypass", status_code=408) from e
 
     logger.info(f"Got webpage: {request.url}")

pyproject.toml

@@ -14,6 +14,7 @@ nodriver = "^0.34"
 requests = "^2.32.3"
 httpx = "^0.27.2"
 pytest-asyncio = "^0.24.0"
+pytest-xdist = "^3.6.1"
 
 
 [build-system]

run_vnc.sh

@@ -0,0 +1,16 @@
+#!/bin/sh
+
+export DISPLAY=:0
+rm -f /tmp/.X0-lock
+
+# Run Xvfb on dispaly 0.
+Xvfb :0 -screen 0 1280x720x16 &
+
+# Run fluxbox windows manager on display 0.
+fluxbox -display :0 &
+
+# Run x11vnc on display 0
+x11vnc -display :0 -forever -ncache 10 &
+
+# Add delay
+sleep 5

src/utils/browser.py

@@ -59,69 +59,70 @@ async def bypass_cloudflare(page: webdriver.Tab):
     """
     challenged = False
     while True:
-        await page
+        await asyncio.sleep(1)
         logger.debug(f"Current page: {page.target.title}")
+
         if page.target.title not in CHALLENGE_TITLES:
             return challenged
+
         if not challenged:
             logger.info("Found challenge")
             challenged = True
 
+        if (
+            page.target.title != "Just a moment..."
+        ):  # If not in cloudflare, wait for autobypass
+            await asyncio.sleep(3)
+            logger.debug("Waiting for challenge to complete")
+            continue
+
         loaded = False
         try:
-            elem = await page.find("lds-ring", timeout=3)
-            parent = elem.parent
-            if not isinstance(parent, Element) or parent.attributes is None:
-                continue
-            for attr in parent.attributes:
-                if attr == "display: none; visibility: hidden;":
-                    loaded = True
-
-        except asyncio.TimeoutError:
-            logger.debug("Challenge loaded")
-        else:
-            if not loaded:
-                logger.debug("Challenge still loading")
-                continue
+            elem = await page.find("lds-ring")
+        except asyncio.TimeoutError as e:
+            logger.error(
+                "Couldn't find lds-ring, probably not a cloudflare challenge, trying again..."
+            )
+            raise InvalidElementError from e
+        if elem is None:
+            logger.error("elem is None")
+            logger.debug(elem)
+            raise InvalidElementError
+
+        parent = elem.parent
+        if not isinstance(parent, Element) or parent.attributes is None:
+            logger.error("parent is not an element or has no attributes")
+            logger.debug(parent)
+            raise InvalidElementError
+
+        for attr in parent.attributes:
+            if attr == "display: none; visibility: hidden;":
+                loaded = True
+                logger.info("Page loaded")
+
+        if not loaded:
+            logger.debug("Challenge still loading")
+            continue
 
-        await page
-        logger.debug("Couldn't find the title, trying other method...")
         elem = await page.find("input")
         elem = elem.parent
         # Get the element containing the shadow root
-
         if isinstance(elem, Element) and elem.shadow_roots:
+            logger.info("Found shadow root")
             inner_elem = Element(elem.shadow_roots[0], page, elem.tree).children[0]
             if isinstance(inner_elem, Element):
+                logger.info("Found elem inside shadow root")
                 logger.debug("Clicking element")
                 await inner_elem.mouse_click()
                 await asyncio.sleep(3)
             else:
                 logger.warning(
-                    "Element is a string, please report this to Byparr dev"
-                )  # I really hope this never happens
-                logger.warning(inner_elem)
+                    "Couldn't find element containing shadow root, trying again..."
+                )
+                logger.debug(inner_elem)
         else:
             logger.warning("Coulnd't find checkbox, trying again...")
-
-
-def get_first_div(elem):
-    """
-    Retrieve the first div element from the given element's children.
-
-    Args:
-    ----
-        elem: The parent element to search for a div child.
-
-    Returns:
-    -------
-        The first div element found, or the original element if no div is found.
-
-    """
-    for child in elem.children:
-        if child.tag_name == "div":
-            return child
-    raise InvalidElementError
+            logger.debug(elem)
 
 
 class InvalidElementError(Exception):

tests/main_test.py

@@ -15,6 +15,7 @@
     "https://extratorrent.st/",  # github is blocking these
     "https://idope.se/",  # github is blocking these
     "https://www.ygg.re/",
+    "https://speed.cd/browse/freeleech",
 ]