feat(cli): Add Adaptive and Stagehand crawler templates (#1888)

Adds three crawler types to the cookiecutter project template and the
scheduled E2E templates tests:

- `adaptive-beautifulsoup` — uses
`AdaptivePlaywrightCrawler.with_beautifulsoup_static_parser()`.
- `adaptive-parsel` — uses
`AdaptivePlaywrightCrawler.with_parsel_static_parser()`.
- `stagehand` — uses `StagehandCrawler`. The template ships a
placeholder API key; the E2E test only validates the build and skips the
run.

Author: vdusek

.github/workflows/on_schedule_tests.yaml

@@ -27,7 +27,7 @@ jobs:
       fail-fast: false
       max-parallel: 12
       matrix:
-        crawler-type: ["playwright_camoufox", "playwright_chrome", "playwright_firefox", "playwright_webkit", "playwright", "parsel", "beautifulsoup"]
+        crawler-type: ["playwright_camoufox", "playwright_chrome", "playwright_firefox", "playwright_webkit", "playwright", "parsel", "beautifulsoup", "adaptive_beautifulsoup", "adaptive_parsel", "stagehand"]
         http-client: ["httpx", "curl_impersonate", "impit"]
         package-manager: ["pip", "uv", "poetry"]
 

src/crawlee/project_template/cookiecutter.json

@@ -1,7 +1,7 @@
 {
     "project_name": "crawlee-python-project",
     "__package_name": "{{ cookiecutter.project_name|lower|replace('-', '_') }}",
-    "crawler_type": ["beautifulsoup", "parsel", "playwright", "playwright-camoufox", "playwright-chrome", "playwright-firefox", "playwright-webkit"],
+    "crawler_type": ["beautifulsoup", "parsel", "adaptive-beautifulsoup", "adaptive-parsel", "playwright", "playwright-camoufox", "playwright-chrome", "playwright-firefox", "playwright-webkit", "stagehand"],
     "__crawler_type": "{{ cookiecutter.crawler_type|lower|replace('-', '_') }}",
     "http_client": ["impit", "httpx", "curl-impersonate"],
     "package_manager": ["poetry", "pip", "uv"],

src/crawlee/project_template/hooks/post_gen_project.py

@@ -1,3 +1,4 @@
+# % set needs_playwright = cookiecutter.crawler_type == 'playwright' or cookiecutter.crawler_type.startswith('adaptive-') or cookiecutter.crawler_type == 'stagehand'
 import platform
 import subprocess
 from pathlib import Path
@@ -12,7 +13,7 @@
 subprocess.check_call(['uv', 'sync'])
 # % endif
 
-# % if cookiecutter.crawler_type == 'playwright'
+# % if needs_playwright
 manager = "{{ cookiecutter.package_manager }}"
 subprocess.check_call([manager, 'run', 'playwright', 'install'])
 # % endif
@@ -38,7 +39,7 @@
     subprocess.check_output([str(path / 'pip'), 'freeze']).decode()
 )
 
-# % if cookiecutter.crawler_type == 'playwright'
+# % if needs_playwright
 subprocess.check_call([str(path / 'playwright'), 'install'])
 # % endif
 # % endif

src/crawlee/project_template/templates/main_adaptive_beautifulsoup.py

@@ -0,0 +1,12 @@
+# % extends 'main.py'
+
+# % block import
+from crawlee.crawlers import AdaptivePlaywrightCrawler
+# % endblock
+
+# % block instantiation
+crawler = AdaptivePlaywrightCrawler.with_beautifulsoup_static_parser(
+    request_handler=router,
+    max_requests_per_crawl=10,
+    {{ self.http_client_instantiation() }})
+# % endblock

src/crawlee/project_template/templates/main_adaptive_parsel.py

@@ -0,0 +1,12 @@
+# % extends 'main.py'
+
+# % block import
+from crawlee.crawlers import AdaptivePlaywrightCrawler
+# % endblock
+
+# % block instantiation
+crawler = AdaptivePlaywrightCrawler.with_parsel_static_parser(
+    request_handler=router,
+    max_requests_per_crawl=10,
+    {{ self.http_client_instantiation() }})
+# % endblock

src/crawlee/project_template/templates/main_stagehand.py

@@ -0,0 +1,23 @@
+# % extends 'main.py'
+
+# % block import
+import os
+
+from crawlee.browsers import StagehandOptions
+from crawlee.crawlers import StagehandCrawler
+# % endblock
+
+# % block instantiation
+model_api_key = os.environ.get('OPENAI_API_KEY')
+if model_api_key is None:
+    raise ValueError('The OPENAI_API_KEY environment variable is not set.')
+
+crawler = StagehandCrawler(
+    request_handler=router,
+    headless=True,
+    max_requests_per_crawl=10,
+    stagehand_options=StagehandOptions(
+        model_api_key=model_api_key,
+    ),
+    {{ self.http_client_instantiation() }})
+# % endblock

src/crawlee/project_template/templates/routes_adaptive_beautifulsoup.py

@@ -0,0 +1,19 @@
+from crawlee.crawlers import AdaptivePlaywrightCrawlingContext
+from crawlee.router import Router
+
+router = Router[AdaptivePlaywrightCrawlingContext]()
+
+
+@router.default_handler
+async def default_handler(context: AdaptivePlaywrightCrawlingContext) -> None:
+    """Default request handler."""
+    context.log.info(f'Processing {context.request.url} ...')
+    title = context.parsed_content.find('title')
+    await context.push_data(
+        {
+            'url': context.request.loaded_url,
+            'title': title.text if title else None,
+        }
+    )
+
+    await context.enqueue_links()

src/crawlee/project_template/templates/routes_adaptive_parsel.py

@@ -0,0 +1,19 @@
+from crawlee.crawlers import AdaptivePlaywrightCrawlingContext
+from crawlee.router import Router
+
+router = Router[AdaptivePlaywrightCrawlingContext]()
+
+
+@router.default_handler
+async def default_handler(context: AdaptivePlaywrightCrawlingContext) -> None:
+    """Default request handler."""
+    context.log.info(f'Processing {context.request.url} ...')
+    title = context.parsed_content.xpath('//title/text()').get()
+    await context.push_data(
+        {
+            'url': context.request.loaded_url,
+            'title': title,
+        }
+    )
+
+    await context.enqueue_links()

src/crawlee/project_template/templates/routes_stagehand.py

@@ -0,0 +1,21 @@
+from crawlee.crawlers import StagehandCrawlingContext
+from crawlee.router import Router
+
+router = Router[StagehandCrawlingContext]()
+
+
+@router.default_handler
+async def default_handler(context: StagehandCrawlingContext) -> None:
+    """Default request handler."""
+    context.log.info(f'Processing {context.request.url} ...')
+
+    data = await context.page.extract(instruction='Get the page title and main heading.')
+
+    await context.push_data(
+        {
+            'url': context.request.loaded_url,
+            'data': data.model_dump(),
+        }
+    )
+
+    await context.enqueue_links()

src/crawlee/project_template/{{cookiecutter.project_name}}/Dockerfile

@@ -1,19 +1,20 @@
 # First, specify the base Docker image.
 # You can see the Docker images from Apify at https://hub.docker.com/r/apify/.
 # You can also use any other image from Docker Hub.
-# % if cookiecutter.crawler_type == 'playwright'
-FROM apify/actor-python-playwright:3.13
+# % if cookiecutter.crawler_type == 'playwright' or cookiecutter.crawler_type.startswith('adaptive-') or cookiecutter.crawler_type == 'stagehand'
+# % set base_image = 'apify/actor-python-playwright:3.13'
 # % elif cookiecutter.crawler_type == 'playwright-camoufox'
-FROM apify/actor-python-playwright-camoufox:3.13
+# % set base_image = 'apify/actor-python-playwright-camoufox:3.13'
 # % elif cookiecutter.crawler_type == 'playwright-chrome'
-FROM apify/actor-python-playwright-chrome:3.13
+# % set base_image = 'apify/actor-python-playwright-chrome:3.13'
 # % elif cookiecutter.crawler_type == 'playwright-firefox'
-FROM apify/actor-python-playwright-firefox:3.13
+# % set base_image = 'apify/actor-python-playwright-firefox:3.13'
 # % elif cookiecutter.crawler_type == 'playwright-webkit'
-FROM apify/actor-python-playwright-webkit:3.13
+# % set base_image = 'apify/actor-python-playwright-webkit:3.13'
 # % else
-FROM apify/actor-python:3.13
+# % set base_image = 'apify/actor-python:3.13'
 # % endif
+FROM {{ base_image }}
 
 RUN apt update && apt install -yq git && rm -rf /var/lib/apt/lists/*