Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 4 additions & 4 deletions packages/browser-crawler/src/internals/browser-crawler.ts
Original file line number Diff line number Diff line change
Expand Up @@ -201,7 +201,7 @@ export interface BrowserCrawlerOptions<
* A hook may optionally return a partial object whose properties are merged into the crawling context,
* allowing the hook to override context members for subsequent hooks and pipeline stages.
*/
preNavigationHooks?: BrowserHook<Context>[];
preNavigationHooks?: BrowserHook<ExtendedContext>[];

/**
* Async functions that are sequentially evaluated after the navigation. Good for checking if the navigation was successful.
Expand All @@ -228,7 +228,7 @@ export interface BrowserCrawlerOptions<
* ]
* ```
*/
postNavigationHooks?: BrowserHook<Context>[];
postNavigationHooks?: BrowserHook<ExtendedContext>[];

/**
* Timeout in which page navigation needs to finish, in seconds.
Expand Down Expand Up @@ -410,8 +410,8 @@ export abstract class BrowserCrawler<

this.launchContext = launchContext;
this.navigationTimeoutMillis = navigationTimeoutSecs * 1000;
this.preNavigationHooks = preNavigationHooks;
this.postNavigationHooks = postNavigationHooks;
this.preNavigationHooks = preNavigationHooks as BrowserHook<Context>[];
this.postNavigationHooks = postNavigationHooks as BrowserHook<Context>[];
this.ignoreIframes = ignoreIframes;
this.ignoreShadowRoots = ignoreShadowRoots;

Expand Down
10 changes: 4 additions & 6 deletions packages/http-crawler/src/internals/http-crawler.ts
Original file line number Diff line number Diff line change
Expand Up @@ -88,7 +88,7 @@ export interface HttpCrawlerOptions<
* ]
* ```
*/
preNavigationHooks?: InternalHttpHook<CrawlingContext>[];
preNavigationHooks?: InternalHttpHook<ExtendedContext>[];

/**
* Async functions that are sequentially evaluated after the navigation. Good for checking if the navigation was successful.
Expand All @@ -107,9 +107,7 @@ export interface HttpCrawlerOptions<
* ]
* ```
*/
postNavigationHooks?: ((
crawlingContext: CrawlingContextWithResponse,
) => Awaitable<void | Partial<CrawlingContextWithResponse>>)[];
postNavigationHooks?: InternalHttpHook<ExtendedContext>[];

/**
* An array of [MIME types](https://developer.mozilla.org/en-US/docs/Web/HTTP/Basics_of_HTTP/MIME_types/Complete_list_of_MIME_types)
Expand Down Expand Up @@ -384,10 +382,10 @@ export class HttpCrawler<
this.ignoreSslErrors = ignoreSslErrors;
this.suggestResponseEncoding = suggestResponseEncoding;
this.forceResponseEncoding = forceResponseEncoding;
this.preNavigationHooks = preNavigationHooks;
this.preNavigationHooks = preNavigationHooks as InternalHttpHook<CrawlingContext>[];
this.postNavigationHooks = [
({ request, response }) => this._abortDownloadOfBody(request, response!),
...postNavigationHooks,
...(postNavigationHooks as InternalHttpHook<CrawlingContextWithResponse>[]),
];

this.saveResponseCookies = saveResponseCookies;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@ import { isDeepStrictEqual } from 'node:util';
import { BasicCrawler } from '@crawlee/basic';
import type { BasicCrawlerOptions, BrowserHook, LoadedRequest, Request } from '@crawlee/browser';
import { extractUrlsFromPage } from '@crawlee/browser';
import type { CheerioCrawlingContext } from '@crawlee/cheerio';
import type { CheerioHook, CheerioCrawlingContext } from '@crawlee/cheerio';
import { CheerioCrawler } from '@crawlee/cheerio';
import type {
ContextPipeline,
Expand Down Expand Up @@ -154,17 +154,20 @@ export interface AdaptivePlaywrightCrawlerContext<
enqueueLinks(options?: EnqueueLinksOptions): Promise<unknown>;
}

interface AdaptiveHookContext extends Pick<AdaptivePlaywrightCrawlerContext, 'id' | 'session' | 'proxyInfo' | 'log'> {
interface AdaptiveHookContext<UserData extends Dictionary = Dictionary> extends Pick<
AdaptivePlaywrightCrawlerContext<UserData>,
'id' | 'session' | 'proxyInfo' | 'log'
> {
page?: Page;
request: Request;
request: Request<UserData>;
gotoOptions?: PlaywrightGotoOptions;
}

interface AdaptiveHook extends BrowserHook<AdaptiveHookContext> {}
type AdaptiveHook<UserData extends Dictionary = Dictionary> = BrowserHook<AdaptiveHookContext<UserData>>;

interface AdaptivePostNavigationHook extends BrowserHook<
Omit<AdaptiveHookContext, 'request'> & { request: LoadedRequest<Request> }
> {}
type AdaptivePostNavigationHook<UserData extends Dictionary = Dictionary> = BrowserHook<
Omit<AdaptiveHookContext<UserData>, 'request'> & { request: LoadedRequest<Request<UserData>> }
>;

export interface AdaptivePlaywrightCrawlerOptions<
ExtendedContext extends AdaptivePlaywrightCrawlerContext = AdaptivePlaywrightCrawlerContext,
Expand All @@ -180,7 +183,7 @@ export interface AdaptivePlaywrightCrawlerOptions<
* A hook may optionally return a partial object whose properties are merged into the crawling context,
* allowing the hook to override context members for subsequent hooks and pipeline stages.
*/
preNavigationHooks?: AdaptiveHook[];
preNavigationHooks?: AdaptiveHook<GetUserDataFromRequest<ExtendedContext['request']>>[];

/**
* Async functions that are sequentially evaluated after the navigation. Good for checking if the navigation was successful.
Expand All @@ -190,7 +193,7 @@ export interface AdaptivePlaywrightCrawlerOptions<
* A hook may optionally return a partial object whose properties are merged into the crawling context
* (e.g. to override `response` after solving a challenge).
*/
postNavigationHooks?: AdaptivePostNavigationHook[];
postNavigationHooks?: AdaptivePostNavigationHook<GetUserDataFromRequest<ExtendedContext['request']>>[];

/**
* Specifies the frequency of rendering type detection checks - 0.1 means roughly 10% of requests.
Expand Down Expand Up @@ -340,16 +343,16 @@ export class AdaptivePlaywrightCrawler<
// `ContextPipeline` handles override merging between hooks for free. The hook signatures
// are structurally compatible with the underlying crawlers' contexts (subset of fields);
// the casts just relax the nominal type difference.
const staticCrawler = new CheerioCrawler({
const staticCrawler = new CheerioCrawler<Dictionary<never>, CheerioCrawlingContext<any, any>>({
...rest,
statisticsOptions: {
persistenceOptions: { enable: false },
},
preNavigationHooks,
postNavigationHooks,
preNavigationHooks: preNavigationHooks as unknown as CheerioHook[],
postNavigationHooks: postNavigationHooks as unknown as CheerioHook[],
});

const browserCrawler = new PlaywrightCrawler({
const browserCrawler = new PlaywrightCrawler<Dictionary<never>, PlaywrightCrawlingContext>({
...rest,
statisticsOptions: {
persistenceOptions: { enable: false },
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -88,7 +88,7 @@ export interface PlaywrightCrawlerOptions<
* ]
* ```
*/
preNavigationHooks?: PlaywrightHook[];
preNavigationHooks?: BrowserHook<ExtendedContext>[];

/**
* Async functions that are sequentially evaluated after the navigation. Good for checking if the navigation was successful.
Expand All @@ -106,7 +106,7 @@ export interface PlaywrightCrawlerOptions<
* ]
* ```
*/
postNavigationHooks?: PlaywrightHook[];
postNavigationHooks?: BrowserHook<ExtendedContext>[];
}

/**
Expand Down
4 changes: 2 additions & 2 deletions packages/puppeteer-crawler/src/internals/puppeteer-crawler.ts
Original file line number Diff line number Diff line change
Expand Up @@ -64,7 +64,7 @@ export interface PuppeteerCrawlerOptions<
* ]
* ```
*/
preNavigationHooks?: PuppeteerHook[];
preNavigationHooks?: BrowserHook<ExtendedContext>[];

/**
* Async functions that are sequentially evaluated after the navigation. Good for checking if the navigation was successful.
Expand All @@ -82,7 +82,7 @@ export interface PuppeteerCrawlerOptions<
* ]
* ```
*/
postNavigationHooks?: PuppeteerHook[];
postNavigationHooks?: BrowserHook<ExtendedContext>[];
}

/**
Expand Down
4 changes: 2 additions & 2 deletions packages/stagehand-crawler/src/internals/stagehand-crawler.ts
Original file line number Diff line number Diff line change
Expand Up @@ -315,12 +315,12 @@ export interface StagehandCrawlerOptions<
/**
* Async functions that are sequentially evaluated before the navigation.
*/
preNavigationHooks?: StagehandHook[];
preNavigationHooks?: BrowserHook<ExtendedContext>[];

/**
* Async functions that are sequentially evaluated after the navigation.
*/
postNavigationHooks?: StagehandHook[];
postNavigationHooks?: BrowserHook<ExtendedContext>[];
}

/**
Expand Down
Loading