Test download failure

scrapy-plugins · Sep 8, 2023 · 57a8192 · 57a8192
1 parent 9843cbb
commit 57a8192
Show file tree

Hide file tree

Showing 3 changed files with 29 additions and 2 deletions.
diff --git a/scrapy_playwright/handler.py b/scrapy_playwright/handler.py
@@ -428,7 +428,7 @@ async def _handle_download(dwnld: Download) -> None:
             self.stats.inc_value("playwright/download_count")
             try:
                 if failure := await dwnld.failure():
-                    raise RuntimeError(failure)
+                    raise RuntimeError(f"Failed to download {dwnld.url}: {failure}")
                 with NamedTemporaryFile() as temp_file:
                     await dwnld.save_as(temp_file.name)
                     temp_file.seek(0)

diff --git a/tests/mockserver.py b/tests/mockserver.py
@@ -71,10 +71,11 @@ def do_GET(self) -> None:
             self.end_headers()
         elif parsed_path.path == "/mancha.pdf":
             body_bytes = (Path(__file__).absolute().parent / "site/files/mancha.pdf").read_bytes()
+            content_length_multiplier = int(query_string.get("content_length_multiplier") or 1)
             self.send_response(200)
             self.send_header("Content-Type", "application/pdf")
             self.send_header("Content-Disposition", 'attachment; filename="mancha.pdf"')
-            self.send_header("Content-Length", str(len(body_bytes)))
+            self.send_header("Content-Length", str(len(body_bytes) * content_length_multiplier))
             self.end_headers()
             self.wfile.write(body_bytes)
         else:

diff --git a/tests/tests_asyncio/test_playwright_requests.py b/tests/tests_asyncio/test_playwright_requests.py
@@ -491,6 +491,32 @@ async def test_download_file_exception(self, NamedTemporaryFile):
                     f" exc_type={type(excinfo.value)} exc_msg={str(excinfo.value)}",
                 ) in self._caplog.record_tuples
 
+    @pytest.mark.asyncio
+    async def test_download_file_failure(self):
+        if self.browser_type != "chromium":
+            pytest.skip()
+
+        async def cancel_download(download):
+            await download.cancel()
+
+        async with make_handler({"PLAYWRIGHT_BROWSER_TYPE": self.browser_type}) as handler:
+            with MockServer() as server:
+                request = Request(
+                    url=server.urljoin("/mancha.pdf?content_length_multiplier=1000"),
+                    meta={
+                        "playwright": True,
+                        "playwright_event_handlers": {"download": cancel_download},
+                    },
+                )
+                with pytest.raises(RuntimeError) as excinfo:
+                    await handler._download_request(request, Spider("foo"))
+                assert (
+                    "scrapy-playwright",
+                    logging.WARNING,
+                    f"Closing page due to failed request: {request}"
+                    f" exc_type={type(excinfo.value)} exc_msg={str(excinfo.value)}",
+                ) in self._caplog.record_tuples
+
 
 class TestCaseChromium(IsolatedAsyncioTestCase, MixinTestCase):
     browser_type = "chromium"