feat(config): let the LightPanda escalation target a renderer that exists

crates/crw-crawl/src/single.rs

@@ -430,12 +430,49 @@ async fn scrape_url_inner(
         // and hide the real outcome behind a fabricated timeout.
         let escalation_budget = deadline.remaining();
         let has_escalation_budget = escalation_budget >= crw_renderer::MIN_TIER_BUDGET;
+        // If the prior tier was lightpanda (returned 200 with thin/no content that
+        // fooled the renderer-level thinness check), escalate to chrome, and when
+        // this deployment runs no Chrome sidecar, to whatever stronger tier it does
+        // have. Pinning the bare literal made the second case a dead end: the pool
+        // rejects a name it does not hold outright, so the escalation failed on the
+        // pin rather than on the page and a configured, healthy tier was never
+        // reached. Chrome stays the first choice so a pool that holds it behaves
+        // exactly as before, including for a host the preference learner has already
+        // promoted to chrome; `auto_ladder_names` supplies the fallback and drops
+        // the tiers the auto chain would not have entered by itself.
+        // `None` means there is nothing above lightpanda, so the escalation is
+        // skipped rather than dispatched: "auto" would re-render the same tier for
+        // the same thin result, and a pin the pool cannot satisfy only produces a
+        // misleading failure.
+        // Otherwise (http tier), let the chain decide so chrome can be reached
+        // through the existing failover path.
+        let escalation_target: Option<&str> = if prior_renderer == Some("lightpanda") {
+            renderer.lightpanda_escalation_target()
+        } else {
+            pinned
+        };
+        let has_escalation_target =
+            escalation_target.is_some() || prior_renderer != Some("lightpanda");
         let should_escalate = (md_is_byte_thin || escalate_for_quality)
             && used_low_tier
             && !js_ladder_exhausted
             && should_escalate_status
             && escalation_eligible
-            && has_escalation_budget;
+            && has_escalation_budget
+            && has_escalation_target;
+        if (md_is_byte_thin || escalate_for_quality)
+            && used_low_tier
+            && should_escalate_status
+            && escalation_eligible
+            && has_escalation_budget
+            && !has_escalation_target
+        {
+            tracing::debug!(
+                url = %req.url,
+                pool = ?renderer.auto_ladder_names(),
+                "skipping JS escalation: no tier above lightpanda in this pool"
+            );
+        }
         if (md_is_byte_thin || escalate_for_quality)
             && used_low_tier
             && should_escalate_status
@@ -450,16 +487,6 @@ async fn scrape_url_inner(
             );
         }
         if should_escalate {
-            // If the prior tier was lightpanda (returned 200 with thin/no content
-            // that fooled the renderer-level thinness check), force chrome on the
-            // escalation. Falling back to "auto" would just hit lightpanda again.
-            // Otherwise (http tier), let the chain decide so chrome can be reached
-            // through the existing failover path.
-            let escalation_target: Option<&str> = if prior_renderer == Some("lightpanda") {
-                Some("chrome")
-            } else {
-                pinned
-            };
             let quality_score_before = md_quality.as_ref().map(|q| q.score);
             tracing::info!(
                 url = %req.url,

crates/crw-renderer/src/lib.rs

@@ -1245,6 +1245,59 @@ impl FallbackRenderer {
         self.js_renderers.iter().map(|r| r.name()).collect()
     }
 
+    /// The JS tiers the auto chain may enter on its own, in construction order.
+    ///
+    /// Drops the two tiers a name-pin would otherwise smuggle past a gate
+    /// `fetch_with_js` applies:
+    ///
+    /// * camoufox held out by `include_in_auto = false` — an internally chosen
+    ///   escalation must not override a deliberate opt-out;
+    /// * chrome_proxy under `auto_egress_escalation`, where the chain lifts it
+    ///   out of the ladder and holds it as a hard-block-gated, load-shed
+    ///   recovery arm. Pinning it by name skips that gate and puts a paid
+    ///   residential render on every page instead of the blocked ones.
+    ///
+    /// This is construction order, not the per-request order: `fetch_with_js`
+    /// reorders for a host the preference learner has promoted, and filters for
+    /// a screenshot. Callers that need one tier should express which one they
+    /// want rather than trusting position alone.
+    pub fn auto_ladder_names(&self) -> Vec<&str> {
+        self.js_renderers
+            .iter()
+            .map(|r| r.name())
+            .filter(|name| *name != "chrome_proxy" || !self.auto_egress_escalation)
+            .filter(|name| {
+                #[cfg(feature = "camoufox")]
+                {
+                    *name != "camoufox" || self.camoufox_in_auto
+                }
+                #[cfg(not(feature = "camoufox"))]
+                {
+                    let _ = name;
+                    true
+                }
+            })
+            .collect()
+    }
+
+    /// Which tier a post-LightPanda escalation should aim at, or `None` when
+    /// this pool has nothing above lightpanda and the escalation should be
+    /// skipped rather than dispatched.
+    ///
+    /// chrome first, so a pool that holds it behaves exactly as the old
+    /// hardcoded target did, including for a host the preference learner has
+    /// promoted to chrome. Otherwise the next tier in [`Self::auto_ladder_names`],
+    /// which is what makes the escalation work on a deployment that runs no
+    /// Chrome sidecar instead of failing on a pin the pool cannot satisfy.
+    pub fn lightpanda_escalation_target(&self) -> Option<&str> {
+        let ladder = self.auto_ladder_names();
+        ladder
+            .iter()
+            .copied()
+            .find(|name| *name == "chrome")
+            .or_else(|| ladder.iter().copied().find(|name| *name != "lightpanda"))
+    }
+
     /// Whether this instance can actually capture a screenshot: at least one
     /// constructed JS renderer speaks CDP `Page.captureScreenshot`. Both the
     /// build features (no CDP feature ⇒ no tier is constructable) and the
@@ -3690,6 +3743,126 @@ mod tests {
         );
     }
 
+    /// The escalation after a thin LightPanda body picks the first non-lightpanda
+    /// entry of `auto_ladder_names`. Pinning a fixed name instead dead-ends on any
+    /// pool that does not hold it, so these four shapes are what that choice has
+    /// to get right.
+    #[cfg(feature = "cdp")]
+    #[test]
+    fn lightpanda_escalation_target_picks_a_tier_the_pool_actually_holds() {
+        let ladder = |cfg: &RendererConfig| {
+            let r =
+                FallbackRenderer::new(cfg, "crw-test", None, &StealthConfig::default()).unwrap();
+            r.lightpanda_escalation_target().map(str::to_string)
+        };
+        let lp = || {
+            Some(CdpEndpoint {
+                ws_url: "ws://127.0.0.1:9222/".into(),
+            })
+        };
+        let ep = |p: &str| {
+            Some(CdpEndpoint {
+                ws_url: format!("ws://127.0.0.1:{p}/"),
+            })
+        };
+
+        // Production shape: chrome present, so the target is unchanged.
+        assert_eq!(
+            ladder(&RendererConfig {
+                mode: RendererMode::Auto,
+                lightpanda: lp(),
+                chrome: ep("9223"),
+                chrome_proxy: ep("9224"),
+                ..Default::default()
+            }),
+            Some("chrome".to_string())
+        );
+
+        // No chrome sidecar: the escalation must reach the tier that IS configured
+        // instead of failing on a pin the pool cannot satisfy.
+        assert_eq!(
+            ladder(&RendererConfig {
+                mode: RendererMode::Auto,
+                lightpanda: lp(),
+                chrome_proxy: ep("9224"),
+                ..Default::default()
+            }),
+            Some("chrome_proxy".to_string())
+        );
+        assert_eq!(
+            ladder(&RendererConfig {
+                mode: RendererMode::Auto,
+                lightpanda: lp(),
+                playwright: ep("9225"),
+                ..Default::default()
+            }),
+            Some("playwright".to_string())
+        );
+
+        // Nothing stronger than lightpanda: no target, so no unsatisfiable pin.
+        assert_eq!(
+            ladder(&RendererConfig {
+                mode: RendererMode::Auto,
+                lightpanda: lp(),
+                ..Default::default()
+            }),
+            None
+        );
+
+        // chrome stays the first choice even when it is not first in construction
+        // order, so a pool that already reached chrome keeps reaching chrome.
+        assert_eq!(
+            ladder(&RendererConfig {
+                mode: RendererMode::Auto,
+                lightpanda: lp(),
+                playwright: ep("9225"),
+                chrome: ep("9223"),
+                ..Default::default()
+            }),
+            Some("chrome".to_string())
+        );
+
+        // Under auto_egress_escalation the chain lifts chrome_proxy out of the
+        // ladder and gates it behind a hard block, so it must not become the
+        // escalation target: pinning it by name would skip that gate and put a
+        // paid residential render on every thin page.
+        assert_eq!(
+            ladder(&RendererConfig {
+                mode: RendererMode::Auto,
+                lightpanda: lp(),
+                chrome_proxy: ep("9224"),
+                auto_egress_escalation: true,
+                ..Default::default()
+            }),
+            None
+        );
+    }
+
+    /// A camoufox held out of the auto chain is constructed and pinnable, but an
+    /// internally chosen escalation must not reach it: naming it bypasses the
+    /// exclusion, which would override a deliberate `include_in_auto = false`.
+    #[cfg(all(feature = "cdp", feature = "camoufox"))]
+    #[test]
+    fn auto_ladder_names_respects_the_camoufox_opt_out() {
+        let with_lp = |include_in_auto: bool| {
+            let mut cfg = camoufox_cfg(RendererMode::Auto, include_in_auto);
+            cfg.lightpanda = Some(CdpEndpoint {
+                ws_url: "ws://127.0.0.1:9222/".into(),
+            });
+            FallbackRenderer::new(&cfg, "crw-test", None, &StealthConfig::default()).unwrap()
+        };
+
+        let held_out = with_lp(false);
+        assert!(
+            held_out.js_renderer_names().contains(&"camoufox"),
+            "the tier is still constructed so an explicit pin can reach it"
+        );
+        assert_eq!(held_out.auto_ladder_names(), vec!["lightpanda"]);
+
+        let in_chain = with_lp(true);
+        assert_eq!(in_chain.auto_ladder_names(), vec!["lightpanda", "camoufox"]);
+    }
+
     /// The HTTP tier decodes every non-PDF body as HTML regardless of its
     /// declared type, so without this gate an empty `application/json` or
     /// `image/*` 2xx would buy a full browser render for nothing.