feat(web-scraper): add support for configurable list of extra modules allowed in extractor scripts

Author: azasypkin

components/retrack-web-scraper/src/api/status/get.test.ts

@@ -22,7 +22,7 @@ await test('[/api/status] returns version from the config', async () => {
       },
     },
     server: { bodyLimit: 5 * 1024 * 1024 },
-
+    extractorSandbox: { extraAllowedModules: [] },
     port: 3,
   };
   const response = await registerStatusGetRoutes(createMock({ config: configMock })).inject({

components/retrack-web-scraper/src/api/web_page/constants.ts

@@ -1,4 +1,4 @@
-import type { RemoteBrowserConfig } from '../../config.js';
+import type { ExtractorSandboxConfig, RemoteBrowserConfig } from '../../config.js';
 
 /**
  * Default timeout for the extractor script, in ms.
@@ -37,6 +37,8 @@ export interface WorkerResultMessage {
 export interface WorkerData {
   // The browser config that the worker should connect Playwright to.
   browserConfig: RemoteBrowserConfig;
+  // The configuration for the extractor sandbox.
+  extractorSandboxConfig: ExtractorSandboxConfig;
   // The extractor script that the worker should execute.
   extractor: string;
   // The parameters that should be passed to the extractor script.

components/retrack-web-scraper/src/api/web_page/execute.test.ts

@@ -1,5 +1,6 @@
 import * as assert from 'node:assert/strict';
 import { test, beforeEach, afterEach } from 'node:test';
+import { configure } from '../../config.js';
 import { createBrowserServerMock } from '../../mocks.js';
 
 import { registerExecuteRoutes } from './execute.js';
@@ -162,6 +163,39 @@ export async function execute(page) {
   assert.strictEqual(response.statusCode, 200);
 });
 
+await test('[/api/web_page/execute] allows extractor scripts to import configured extra modules', async (t) => {
+  t.mock.method(Date, 'now', () => 123000);
+
+  const response = await registerExecuteRoutes(
+    createMock({
+      wsEndpoint: browserServerMock.endpoint,
+      config: {
+        ...configure(),
+        extractorSandbox: { extraAllowedModules: ['node:fs'] },
+        browser: { chromium: { protocol: 'cdp', backend: 'chromium', wsEndpoint: browserServerMock.endpoint } },
+      },
+    }),
+  ).inject({
+    method: 'POST',
+    url: '/api/web_page/execute',
+    payload: {
+      extractor: `
+export async function execute() {
+  await import('timers');
+  await import('node:fs');
+  return 'OK';
+};
+  `
+        .replaceAll('\n', '')
+        .trim(),
+      tags: [],
+    },
+  });
+
+  assert.strictEqual(response.body, JSON.stringify('OK'));
+  assert.strictEqual(response.statusCode, 200);
+});
+
 await test('[/api/web_page/execute] allows extractor scripts to import `data:` modules', async (t) => {
   t.mock.method(Date, 'now', () => 123000);
 

components/retrack-web-scraper/src/api/web_page/execute.ts

@@ -119,6 +119,7 @@ export function registerExecuteRoutes({ config, server, getLocalBrowserServer }:
       const workerLog = logger.child({ provider: 'worker' });
       const workerData: WorkerData = {
         browserConfig: remoteBrowserConfig,
+        extractorSandboxConfig: config.extractorSandbox,
         extractor: request.body.extractor,
         extractorParams: request.body.extractorParams,
         tags: request.body.tags,

components/retrack-web-scraper/src/api/web_page/extractor_module_hooks.ts

@@ -1,7 +1,31 @@
-import type { ResolveFnOutput, ResolveHookContext } from 'module';
+import type { ResolveFnOutput, ResolveHookContext, InitializeHook } from 'module';
+import type { ExtractorSandboxConfig } from '../../config.js';
 
 // This set contains the modules that are allowed to be imported by extractor scripts.
-const EXTRACTOR_MODULE_ALLOWLIST = new Set(['node:util', 'stream', 'stream/promises']);
+const EXTRACTOR_MODULE_ALLOWLIST = new Set([
+  'node:stream',
+  'node:stream/promises',
+  'stream',
+  'stream/promises',
+
+  'node:timers',
+  'node:timers/promises',
+  'timers',
+  'timers/promises',
+
+  'node:util',
+  'util',
+]);
+
+// The initialize hook provides a way to define a custom function that runs in
+// the hooks thread when the hooks module is initialized. Initialization happens
+// when the hooks module is registered via `register`.
+export const initialize: InitializeHook<ExtractorSandboxConfig> = async ({ extraAllowedModules }) => {
+  // Add extra allowed modules to the allowlist.
+  for (const module of extraAllowedModules) {
+    EXTRACTOR_MODULE_ALLOWLIST.add(module);
+  }
+};
 
 // This hook is called whenever a module is resolved, allowing you to intercept the resolution process and prevent
 // extractor scripts from importing modules. This is useful for preventing extractor scripts from accessing sensitive

components/retrack-web-scraper/src/api/web_page/worker.ts

@@ -3,6 +3,7 @@ import { register } from 'node:module';
 import { pathToFileURL } from 'node:url';
 import { resolve } from 'node:path';
 import type { Browser, Page } from 'playwright-core';
+import type { ExtractorSandboxConfig } from '../../config.js';
 
 import { Diagnostics } from '../diagnostics.js';
 import type { WorkerData } from './constants.js';
@@ -17,6 +18,7 @@ if (!parentPort) {
 // Load the extractor script as an ES module.
 const {
   browserConfig,
+  extractorSandboxConfig,
   extractor,
   extractorParams,
   tags,
@@ -50,7 +52,9 @@ for (const Class of [
 
 // SECURITY: We load custom hooks to prevent extractor scripts from importing sensitive native and playwright modules.
 // See https://github.com/nodejs/node/issues/47747 for more details.
-register(resolve(import.meta.dirname, './extractor_module_hooks.js'), pathToFileURL('./'));
+register<ExtractorSandboxConfig>(resolve(import.meta.dirname, './extractor_module_hooks.js'), pathToFileURL('./'), {
+  data: extractorSandboxConfig,
+});
 const extractorModule = (await import(`data:text/javascript,${encodeURIComponent(extractor)}`)) as {
   execute: (page: Page, context: { tags: string[]; params?: unknown; previousContent?: unknown }) => Promise<unknown>;
 };

components/retrack-web-scraper/src/config.ts

@@ -8,6 +8,7 @@ export interface Config {
   isDev: boolean;
   logLevel: string;
   browser: { screenshotsPath?: string; chromium?: BrowserConfig; firefox?: BrowserConfig };
+  extractorSandbox: ExtractorSandboxConfig;
   userAgent?: string;
   server: { bodyLimit: number };
 }
@@ -39,6 +40,13 @@ export interface LocalBrowserConfig {
   chromiumSandbox: boolean;
 }
 
+// Represents the configuration for the extractor sandbox.
+export interface ExtractorSandboxConfig {
+  // The list of Node.js modules that are allowed to be imported in the
+  // extractor scripts in addition to those that are allowed by default.
+  extraAllowedModules: string[];
+}
+
 export function configure(): Config {
   return {
     version: pkg.version,
@@ -76,5 +84,10 @@ export function configure(): Config {
       bodyLimit: +(process.env.RETRACK_WEB_SCRAPER_SERVER_BODY_LIMIT ?? 0) || 5 * 1024 * 1024,
     },
     userAgent: process.env.RETRACK_WEB_SCRAPER_USER_AGENT,
+    extractorSandbox: {
+      extraAllowedModules: process.env.RETRACK_WEB_SCRAPER_EXTRACTOR_SANDBOX_EXTRA_ALLOWED_MODULES
+        ? process.env.RETRACK_WEB_SCRAPER_EXTRACTOR_SANDBOX_EXTRA_ALLOWED_MODULES.split(',').map((m) => m.trim())
+        : [],
+    },
   };
 }