fix: cancel requests only in standby (#68)

* Revert "chore: Revert "fix: cancel timed out requests (#65)" (#67)"

This reverts commit 7d66686.

* only cancel requests for standby actors

* Update CHANGELOG.md

* Update CHANGELOG.md

Author: MQ37

CHANGELOG.md

@@ -1,5 +1,15 @@
 This changelog summarizes all changes of the RAG Web Browser
 
+### 1.0.15 (2025-03-27)
+
+🐛 Bug Fixes
+- Cancel requests only in standby mode
+
+### 1.0.13 (2025-03-27)
+
+🐛 Bug Fixes
+- Cancel crawling requests from timed-out search queries
+
 ### 1.0.12 (2025-03-24)
 
 🐛 Bug Fixes

src/main.ts

@@ -10,6 +10,7 @@ import { RagWebBrowserServer } from './mcp/server.js';
 import { addTimeoutToAllResponses } from './responses.js';
 import { handleSearchRequest, handleSearchNormalMode } from './search.js';
 import { Input } from './types.js';
+import { isActorStandby } from './utils.js';
 
 await Actor.init();
 
@@ -57,10 +58,9 @@ app.use((req, res) => {
     res.status(404).json({ message: `The is nothing at route ${req.method} ${req.originalUrl}. ${HELP_MESSAGE}` });
 });
 
-const standbyMode = Actor.getEnv().metaOrigin === 'STANDBY';
 const originalInput = await Actor.getInput<Partial<Input>>() ?? {} as Input;
 
-if (standbyMode) {
+if (isActorStandby()) {
     log.info('Actor is running in the STANDBY mode.');
 
     const host = Actor.isAtHome() ? process.env.ACTOR_STANDBY_URL : 'http://localhost';

src/request-handler.ts

@@ -3,9 +3,9 @@ import { load } from 'cheerio';
 import { CheerioCrawlingContext, htmlToText, log, PlaywrightCrawlingContext, sleep, Request } from 'crawlee';
 
 import { ContentCrawlerStatus, ContentCrawlerTypes } from './const.js';
-import { addResultToResponse, sendResponseIfFinished } from './responses.js';
+import { addResultToResponse, responseData, sendResponseIfFinished } from './responses.js';
 import { Output, ContentCrawlerUserData } from './types.js';
-import { addTimeMeasureEvent, transformTimeMeasuresToRelative } from './utils.js';
+import { addTimeMeasureEvent, isActorStandby, transformTimeMeasuresToRelative } from './utils.js';
 import { processHtml } from './website-content-crawler/html-processing.js';
 import { htmlToMarkdown } from './website-content-crawler/markdown.js';
 
@@ -27,6 +27,22 @@ async function waitForPlaywright({ page }: PlaywrightCrawlingContext, time: numb
     return Promise.race([page.waitForLoadState('networkidle', { timeout: 0 }), sleep(time - hardDelay)]);
 }
 
+/**
+ * Checks if the request should time out based on response timeout.
+ * It verifies if the response data contains the responseId. If not, it sets the request's noRetry flag
+ * to true and throws an error to cancel the request.
+ *
+ * @param {Request} request - The request object to be checked.
+ * @param {string} responseId - The response ID to look for in the response data.
+ * @throws {Error} Throws an error if the request times out.
+ */
+function checkTimeoutAndCancelRequest(request: Request, responseId: string) {
+    if (!responseData.has(responseId)) {
+        request.noRetry = true;
+        throw new Error('Timed out. Cancelling the request...');
+    }
+}
+
 /**
  * Decide whether to wait based on the remaining time left for the Actor to run.
  * Always waits if the Actor is in the STANDBY_MODE.
@@ -148,7 +164,9 @@ export async function requestHandlerPlaywright(
     context: PlaywrightCrawlingContext<ContentCrawlerUserData>,
 ) {
     const { request, response, page, closeCookieModals } = context;
-    const { contentScraperSettings: settings } = request.userData;
+    const { contentScraperSettings: settings, responseId } = request.userData;
+
+    if (isActorStandby()) checkTimeoutAndCancelRequest(request, responseId);
 
     log.info(`Processing URL: ${request.url}`);
     addTimeMeasureEvent(request.userData, 'playwright-request-start');
@@ -180,6 +198,9 @@ export async function requestHandlerCheerio(
     context: CheerioCrawlingContext<ContentCrawlerUserData>,
 ) {
     const { $, request, response } = context;
+    const { responseId } = request.userData;
+
+    if (isActorStandby()) checkTimeoutAndCancelRequest(request, responseId);
 
     log.info(`Processing URL: ${request.url}`);
     addTimeMeasureEvent(request.userData, 'cheerio-request-start');

src/responses.ts

@@ -11,7 +11,7 @@ type ResponseData = {
     timeoutId?: NodeJS.Timeout;
 };
 
-const responseData = new Map<string, ResponseData>();
+export const responseData = new Map<string, ResponseData>();
 
 /**
  * Helper function to get response object by responseId.
@@ -39,7 +39,7 @@ export function createResponsePromise(responseId: string, timeoutSecs: number):
 
         // Set a timeout to reject the promise if it takes too long
         data.timeoutId = setTimeout(() => {
-            sendResponseError(responseId, 'Timed out');
+            sendResponseError(responseId, 'Timed out.');
         }, timeoutSecs * 1000);
     });
 }

src/utils.ts

@@ -1,10 +1,15 @@
+import { Actor } from 'apify';
 import { RequestOptions, log, ProxyConfiguration } from 'crawlee';
 import { parse, ParsedUrlQuery } from 'querystring';
 
 import { defaults } from './const.js';
 import { OrganicResult, ContentScraperSettings, TimeMeasure, ContentCrawlerUserData, SearchCrawlerUserData } from './types.js';
 import inputSchema from '../.actor/input_schema.json' with { type: 'json' };
 
+export function isActorStandby(): boolean {
+    return Actor.getEnv().metaOrigin === 'STANDBY';
+}
+
 export function parseParameters(url: string): ParsedUrlQuery {
     const params = parse(url.slice(1));