Squashed commit of the following:

commit 3eec08f
Author: Adam Thompson <2414030+TheSonOfThomp@users.noreply.github.com>
Date:   Fri May 9 10:46:31 2025 -0400

    Ingest with recursive crawler (#2849)

    * feat(ui): initialize chatbot UI with React and Vite (#2841)

    * feat(ui): initialize chatbot UI with React and Vite

    - Added package.json for UI package with scripts and dependencies.
    - Created App.css for styling the main application layout.
    - Implemented App.tsx as the main component integrating the MongoDB Chatbot UI.
    - Added a React SVG asset for branding.
    - Created index.css for global styles and theming.
    - Set up main.tsx as the entry point for the React application.
    - Added vite-env.d.ts for Vite type definitions.
    - Configured TypeScript with tsconfig.json and tsconfig.node.json for the UI package.
    - Created vite.config.ts for Vite configuration with React plugin.

    * fix: update license to Apache-2.0 and refactor import statements for consistency

    * feat(ui): initialize chatbot UI with React and Vite

    - Added package.json for UI package with scripts and dependencies.
    - Created App.css for styling the main application layout.
    - Implemented App.tsx as the main component integrating the MongoDB Chatbot UI.
    - Added a React SVG asset for branding.
    - Created index.css for global styles and theming.
    - Set up main.tsx as the entry point for the React application.
    - Added vite-env.d.ts for Vite type definitions.
    - Configured TypeScript with tsconfig.json and tsconfig.node.json for the UI package.
    - Created vite.config.ts for Vite configuration with React plugin.

    * fix: update license to Apache-2.0 and refactor import statements for consistency

    * feat(ingest): add initial configuration and data sources for chatbot ingestion

    * update configs

    * feat(ingest): implement data sources for LeafyGreen UI and MongoDB Chatbot Framework

    * chore: update pnpm workspace configuration to include 'apps/*' directory

    * feat(ingest): add data sources for LeafyGreen UI and MongoDB Chatbot Framework, and refactor Azure OpenAI embedder

    * feat(ingest): add MongoDB Design website data source and integrate into ingest configuration

    * Delete package-lock.json

    * reset ui

    * feat(crawler): export LoadedPageContents and ProcessSingleUrl interfaces

    * feat(crawler): refactor crawling logic to use processLangchainDocument and update allowedDomains format

    * feat(crawler): update recursive crawl logic to process documents with processLangchainDocument

    * feat(crawler): update package.json exports and refactor lodash imports

    * feat(crawler): implement createWebSourceConstructor and update ingest configuration

    * Update ingest.config.ts

    * feat(crawler): replace createWebSourceConstructor with webSourceConstructor and update data sources

    * fix(crawler): handle invalid URLs in newURL and recursive crawl logic

    * Resolve tsconfig.json

    ---------

    Co-authored-by: Terrence Keane <terrence.keane@mongodb.com>

Author: TheSonOfThomp

apps/chatbot-server/src/utils/webSourceConstructor.ts

@@ -0,0 +1,56 @@
+/* eslint-disable no-console */
+import { recursiveCrawlFromBaseURL } from '@lg-tools/crawler';
+import { Page } from 'mongodb-rag-core';
+import { type DataSource } from 'mongodb-rag-core/dataSources';
+
+export interface WebSourceConstructorOptions {
+  maxDepth?: number;
+  verbose?: boolean;
+}
+
+/**
+ * Returns a constructor function
+ * that creates a web source
+ */
+export async function webSourceConstructor(
+  source: string,
+  options?: WebSourceConstructorOptions,
+): Promise<DataSource> {
+  const { maxDepth = 3, verbose = false } = {
+    maxDepth: 3,
+    verbose: false,
+    ...options,
+  };
+  return {
+    name: source,
+    fetchPages: async () => {
+      verbose && console.log(`🐶 Fetching source ${source}`);
+      const pages: Array<Page> = [];
+
+      await recursiveCrawlFromBaseURL(
+        ({ document, title, href }) => {
+          verbose && console.log(`🪲 Crawled page ${title} - ${href}`);
+          pages.push({
+            url: href,
+            title,
+            body: document.pageContent,
+            format: 'txt',
+            sourceName: source,
+            metadata: {
+              id: document.id,
+              ...document.metadata,
+            },
+          });
+        },
+        {
+          baseUrl: source,
+          maxDepth,
+          verbose,
+          enableRecursion: true,
+        },
+      );
+
+      return pages;
+    },
+  };
+}