fix(community): URL encode paths in GitHub document loader (#8860)

Co-authored-by: Dylan Dignan <dylan.dignan@epicgames.com>

Author: 2 people

libs/langchain-community/src/document_loaders/tests/github.test.ts

@@ -61,3 +61,56 @@ describe("GithubRepoLoader recursion", () => {
     ).toThrow();
   });
 });
+
+describe("GithubRepoLoader URL encoding", () => {
+  test("Should properly encode special characters in directory paths", async () => {
+    // Mock fetch to capture the URLs being called
+    const mockFetch = jest.fn().mockImplementation((url) => {
+      // Check that special characters are properly encoded in the URL
+      expect(url).toContain("src%2Fapp%2F%255Fmeta"); // The full encoded path
+
+      return Promise.resolve({
+        ok: true,
+        json: () =>
+          Promise.resolve([
+            {
+              name: "%5Fmeta",
+              path: "src/app/%5Fmeta",
+              type: "dir",
+              size: 0,
+              url: "https://api.github.com/repos/test/test/contents/src/app/%5Fmeta",
+              html_url: "",
+              sha: "abc123",
+              git_url: "",
+              download_url: "",
+              _links: {
+                self: "",
+                git: "",
+                html: "",
+              },
+            },
+          ]),
+      });
+    });
+
+    global.fetch = mockFetch as any;
+
+    const loader = new GithubRepoLoader(
+      "https://github.com/test/test/tree/main/src/app/%5Fmeta",
+      {
+        branch: "main",
+        recursive: false,
+        unknown: "warn",
+      }
+    );
+
+    // This should call fetchRepoFiles with "src/app/%5Fmeta" path
+    await loader.load();
+
+    // Verify that fetch was called with properly encoded URL
+    expect(mockFetch).toHaveBeenCalledWith(
+      expect.stringContaining("contents/src%2Fapp%2F%255Fmeta"),
+      expect.any(Object)
+    );
+  });
+});

libs/langchain-community/src/document_loaders/web/github.ts

@@ -647,7 +647,9 @@ export class GithubRepoLoader
    * @returns A promise that resolves to an array of GithubFile instances.
    */
   private async fetchRepoFiles(path: string): Promise<GithubFile[]> {
-    const url = `${this.apiUrl}/repos/${this.owner}/${this.repo}/contents/${path}?ref=${this.branch}`;
+    const url = `${this.apiUrl}/repos/${this.owner}/${
+      this.repo
+    }/contents/${encodeURIComponent(path)}?ref=${this.branch}`;
     return this.caller.call(async () => {
       this.log(`Fetching ${url}`);
       const response = await fetch(url, { headers: this.headers });