fix: fix page_options for PlaywrightBrowserPlugin (#796)

### Description

- fix  `page_options` for `PlaywrightBrowserPlugin`

### Issues

- Closes: #755, #751 

### Testing

- Add test for check workability `page_options` in
`PlaywrightBrowserPlugin`

### Checklist

- [ ] CI passed

Author: Mantisus

src/crawlee/browsers/_browser_pool.py

@@ -112,9 +112,9 @@ def with_default_plugin(
             browser_options: Keyword arguments to pass to the browser launch method. These options are provided
                 directly to Playwright's `browser_type.launch` method. For more details, refer to the Playwright
                 documentation: https://playwright.dev/python/docs/api/class-browsertype#browser-type-launch.
-            page_options: Keyword arguments to pass to the new page method. These options are provided directly to
-                Playwright's `browser_context.new_page` method. For more details, refer to the Playwright documentation:
-                https://playwright.dev/python/docs/api/class-browsercontext#browser-context-new-page.
+            page_options: Keyword arguments to pass to the page object is set at the playwright context level.
+                These options are provided directly to Playwright's `browser.new_context` method. For more details,
+                refer to the Playwright documentation: https://playwright.dev/python/docs/api/class-browser#browser-new-context.
             headless: Whether to run the browser in headless mode.
             kwargs: Additional arguments for default constructor.
         """

src/crawlee/browsers/_playwright_browser_controller.py

@@ -98,13 +98,12 @@ async def new_page(
         proxy_info: ProxyInfo | None = None,
     ) -> Page:
         if not self._browser_context:
-            self._browser_context = await self._create_browser_context(proxy_info)
+            self._browser_context = await self._create_browser_context(page_options, proxy_info)
 
         if not self.has_free_capacity:
             raise ValueError('Cannot open more pages in this browser.')
 
-        page_options = dict(page_options) if page_options else {}
-        page = await self._browser_context.new_page(**page_options)
+        page = await self._browser_context.new_page()
 
         # Handle page close event
         page.on(event='close', f=self._on_page_close)
@@ -130,17 +129,20 @@ def _on_page_close(self, page: Page) -> None:
         """Handle actions after a page is closed."""
         self._pages.remove(page)
 
-    async def _create_browser_context(self, proxy_info: ProxyInfo | None = None) -> BrowserContext:
+    async def _create_browser_context(
+        self, page_options: Mapping[str, Any] | None = None, proxy_info: ProxyInfo | None = None
+    ) -> BrowserContext:
         """Create a new browser context with the specified proxy settings."""
         if self._header_generator:
             common_headers = self._header_generator.get_common_headers()
             sec_ch_ua_headers = self._header_generator.get_sec_ch_ua_headers(browser_type=self.browser_type)
             user_agent_header = self._header_generator.get_user_agent_header(browser_type=self.browser_type)
             extra_http_headers = dict(common_headers | sec_ch_ua_headers | user_agent_header)
-            user_agent = user_agent_header.get('User-Agent')
         else:
             extra_http_headers = None
-            user_agent = None
+
+        page_options = dict(page_options) if page_options else {}
+        page_options['extra_http_headers'] = page_options.get('extra_http_headers', extra_http_headers)
 
         proxy = (
             ProxySettings(
@@ -152,8 +154,4 @@ async def _create_browser_context(self, proxy_info: ProxyInfo | None = None) ->
             else None
         )
 
-        return await self._browser.new_context(
-            user_agent=user_agent,
-            extra_http_headers=extra_http_headers,
-            proxy=proxy,
-        )
+        return await self._browser.new_context(proxy=proxy, **page_options)

src/crawlee/browsers/_playwright_browser_plugin.py

@@ -46,9 +46,9 @@ def __init__(
             browser_options: Keyword arguments to pass to the browser launch method. These options are provided
                 directly to Playwright's `browser_type.launch` method. For more details, refer to the Playwright
                 documentation: https://playwright.dev/python/docs/api/class-browsertype#browser-type-launch.
-            page_options: Keyword arguments to pass to the new page method. These options are provided directly to
-                Playwright's `browser_context.new_page` method. For more details, refer to the Playwright documentation:
-                https://playwright.dev/python/docs/api/class-browsercontext#browser-context-new-page.
+            page_options: Keyword arguments to pass to the page object is set at the playwright context level.
+                These options are provided directly to Playwright's `browser.new_context` method. For more details,
+                refer to the Playwright documentation: https://playwright.dev/python/docs/api/class-browser#browser-new-context.
             max_open_pages_per_browser: The maximum number of pages that can be opened in a single browser instance.
                 Once reached, a new browser instance will be launched to handle the excess.
         """

tests/unit/browsers/test_browser_pool.py

@@ -151,3 +151,12 @@ async def test_methods_raise_error_when_not_active() -> None:
 
     async with browser_pool:
         assert browser_pool.active is True
+
+
+async def test_with_plugin_contains_page_options(httpbin: URL) -> None:
+    plugin = PlaywrightBrowserPlugin(page_options={'user_agent': 'My Best User-Agent'})
+    async with BrowserPool(plugins=[plugin]) as browser_pool:
+        test_page = await browser_pool.new_page()
+        await test_page.page.goto(str(httpbin / 'user-agent'))
+        assert 'My Best User-Agent' in await test_page.page.content()
+        await test_page.page.close()