Scrape all pages, not only level 1 navigation (langchain-ai#1191)

* Use sitemap to properly load all paths

Gitbook's server-side rendering means that when we fetch the contents of the index page
the sub-menu items are not rendered and thus are invisible to $('nav a').
This patch makes the loader use sitemap.xml which has an index of all pages.

Inspired by the Python version.

* Fix lint issues

* Fix the logic

* Add the integration test provided by jacoblee93

* Fix loading the docs

* Fix formatting

langchain/src/document_loaders/tests/gitbook.int.test.ts

@@ -0,0 +1,19 @@
+import { test } from "@jest/globals";
+import { GitbookLoader } from "../web/gitbook.js";
+
+test("Test GitbookLoader", async () => {
+  const loader = new GitbookLoader(
+    "https://docs.gitbook.com/product-tour/navigation"
+  );
+
+  const docs = await loader.load();
+  console.log("Loaded", docs.length, "Gitbook documents");
+});
+
+test.only("Test GitbookLoader with shouldLoadAllPaths", async () => {
+  const loader = new GitbookLoader("https://docs.maildrop.cc", {
+    shouldLoadAllPaths: true,
+  });
+  const docs = await loader.load();
+  console.log("Loaded", docs.length, "Gitbook documents");
+});

langchain/src/document_loaders/web/gitbook.ts

@@ -10,7 +10,12 @@ export class GitbookLoader extends CheerioWebBaseLoader {
   shouldLoadAllPaths = false;
 
   constructor(public webPath: string, params: GitbookLoaderParams = {}) {
-    super(webPath);
+    const path =
+      params.shouldLoadAllPaths === true ? `${webPath}/sitemap.xml` : webPath;
+    super(path);
+
+    this.webPath = path;
+
     this.shouldLoadAllPaths =
       params.shouldLoadAllPaths ?? this.shouldLoadAllPaths;
   }
@@ -45,14 +50,12 @@ export class GitbookLoader extends CheerioWebBaseLoader {
   }
 
   private async loadAllPaths($: CheerioAPI): Promise<Document[]> {
-    const relative_paths = $("nav a")
+    const urls = $("loc")
       .toArray()
-      .map((element) => $(element).attr("href"))
-      .filter((text) => text && text[0] === "/");
+      .map((element) => $(element).text());
 
     const documents: Document[] = [];
-    for (const path of relative_paths) {
-      const url = this.webPath + path;
+    for (const url of urls) {
       console.log(`Fetching text from ${url}`);
       const html = await GitbookLoader._scrape(url, this.caller, this.timeout);
       documents.push(...this.loadPath(html, url));