Allow maxPagesToCrawl to be optional and infinite by setting 0 which …

…will display the infinity symbol. Default is 50

README.md

@@ -74,7 +74,8 @@ type Config = {
   match: string;
   /** Selector to grab the inner text from */
   selector: string;
-  /** Don't crawl more than this many pages */
+
+  /** Optional - Don't crawl more than this many pages (0 = Crawl all, Default = 50)*/
   maxPagesToCrawl: number;
   /** File name for the finished data */
   outputFileName: string;

src/config.ts

@@ -26,10 +26,11 @@ export const configSchema = z.object({
    */
   selector: z.string().optional(),
   /**
+   * **Optional:**
    * Don't crawl more than this many pages
    * @default 50
    */
-  maxPagesToCrawl: z.number().int().positive(),
+  maxPagesToCrawl: z.number().int().nonnegative().or(z.undefined()).optional(),
   /**
    * File name for the finished data
    * @default "output.json"

src/core.ts

@@ -55,6 +55,12 @@ export async function crawl(config: Config) {
     const crawler = new PlaywrightCrawler({
       // Use the requestHandler to process each of the crawled pages.
       async requestHandler({ request, page, enqueueLinks, log, pushData }) {
+        // Warn if unlimited crawling is enabled
+        if (config.maxPagesToCrawl == 0) {
+          config.maxPagesToCrawl = undefined;
+          log.warningOnce(`maxPagesToCrawl is set to ${config.maxPagesToCrawl} which means it will contine until it cannot find anymore links defined by match: ${config.match}`);          
+        }
+
         if (config.cookie) {
           // Set the cookie for the specific URL
           const cookie = {
@@ -66,9 +72,11 @@ export async function crawl(config: Config) {
         }
 
         const title = await page.title();
+        // Display the pageCounter/maxPagesToCrawl number or pageCounter/∞ if maxPagesToCrawl=0
+        const maxPagesToCrawlDisplay = config.maxPagesToCrawl == undefined ? "∞" : config.maxPagesToCrawl;
         pageCounter++;
         log.info(
-          `Crawling: Page ${pageCounter} / ${config.maxPagesToCrawl} - URL: ${request.loadedUrl}...`,
+          `Crawling: Page ${pageCounter} / ${maxPagesToCrawlDisplay} - URL: ${request.loadedUrl}...`
         );
 
         // Use custom handling for XPath selector