Implement IPFS multiple gateway support with automatic fallback

packages/dappmanager/src/index.ts

@@ -8,7 +8,7 @@ import {
   copyHostServices,
   copyHostTimers
 } from "@dappnode/hostscriptsservices";
-import { DappnodeInstaller, getIpfsUrl, postRestartPatch } from "@dappnode/installer";
+import { DappnodeInstaller, getIpfsUrls, postRestartPatch } from "@dappnode/installer";
 import * as calls from "./calls/index.js";
 import { routesLogger, subscriptionsLogger, logs } from "@dappnode/logger";
 import * as routes from "./api/routes/index.js";
@@ -47,11 +47,11 @@ initializeDb()
   .then(() => logs.info("Initialized Database"))
   .catch((e) => logs.error("Error inititializing Database", e));
 
-let ipfsUrl = params.IPFS_LOCAL;
+let ipfsUrls = [params.IPFS_LOCAL];
 try {
-  ipfsUrl = getIpfsUrl(); // Attempt to update with value from getIpfsUrl
+  ipfsUrls = getIpfsUrls(); // Attempt to update with value from getIpfsUrls
 } catch (e) {
-  logs.error(`Error getting ipfsUrl: ${e.message}. Using default: ${ipfsUrl}`);
+  logs.error(`Error getting ipfsUrls: ${e.message}. Using default: ${ipfsUrls}`);
 }
 
 // Read and print version data
@@ -75,7 +75,7 @@ const providers = new MultiUrlJsonRpcProvider(
 
 // Required db to be initialized
 export const directory = new DappNodeDirectory(providers);
-export const dappnodeInstaller = new DappnodeInstaller(ipfsUrl, providers);
+export const dappnodeInstaller = new DappnodeInstaller(ipfsUrls, providers);
 
 export const publicRegistry = new DappNodeRegistry("public");
 

packages/installer/src/dappnodeInstaller.ts

@@ -30,29 +30,43 @@ import { omit } from "lodash-es";
 import { JsonRpcApiProvider } from "ethers";
 
 /**
- * Returns the ipfsUrl to initialize the ipfs instance
+ * Returns the ipfsUrls array to initialize the IPFS instance
+ * Local IPFS comes first as default, then remote as fallback
  */
-export function getIpfsUrl(): string {
-  // Fort testing
-  if (params.IPFS_HOST) return params.IPFS_HOST;
+export function getIpfsUrls(): string[] {
+  // For testing
+  if (params.IPFS_HOST) return [params.IPFS_HOST];
 
   const ipfsClientTarget = db.ipfsClientTarget.get();
   if (!ipfsClientTarget) throw Error("Ipfs client target is not set");
+
   // local
-  if (ipfsClientTarget === IpfsClientTarget.local) return params.IPFS_LOCAL;
+  if (ipfsClientTarget === IpfsClientTarget.local) {
+    return [params.IPFS_LOCAL, params.IPFS_REMOTE]; // Local first, remote fallback
+  }
   // remote
-  return db.ipfsGateway.get();
+  const remoteGateway = db.ipfsGateway.get();
+  return [remoteGateway, params.IPFS_LOCAL]; // Remote first, local fallback
+}
+
+/**
+ * Returns the ipfsUrl to initialize the ipfs instance
+ * @deprecated Use getIpfsUrls() instead for multiple gateway support
+ */
+export function getIpfsUrl(): string {
+  const urls = getIpfsUrls();
+  return urls[0]; // Return first URL for backward compatibility
 }
 
 export class DappnodeInstaller extends DappnodeRepository {
-  constructor(ipfsUrl: string, provider: JsonRpcApiProvider) {
-    super(ipfsUrl, provider);
+  constructor(ipfsUrls: string | string[], provider: JsonRpcApiProvider) {
+    super(ipfsUrls, provider);
   }
 
   private async updateProviders(): Promise<void> {
-    const newIpfsUrl = getIpfsUrl();
+    const newIpfsUrls = getIpfsUrls();
     // super.changeEthProvider();
-    super.changeIpfsGatewayUrl(newIpfsUrl);
+    super.changeIpfsGatewayUrl(newIpfsUrls);
   }
 
   /**

packages/installer/src/index.ts

@@ -1,5 +1,5 @@
 export * from "./dappGet/index.js";
 export * from "./installer/index.js";
-export { DappnodeInstaller, getIpfsUrl } from "./dappnodeInstaller.js";
+export { DappnodeInstaller, getIpfsUrl, getIpfsUrls } from "./dappnodeInstaller.js";
 // calls
 export * from "./calls/index.js";

packages/toolkit/src/repository/repository.ts

@@ -39,25 +39,28 @@ const source = "ipfs" as const;
  * @extends ApmRepository
  */
 export class DappnodeRepository extends ApmRepository {
-  protected gatewayUrl: string;
+  protected gatewayUrls: string[];
   protected localIpfsUrl = "http://ipfs.dappnode:5001";
 
   /**
    * Constructs an instance of DappnodeRepository
-   * @param ipfsUrl - The URL of the IPFS network node.
-   * @param ethUrl - The URL of the Ethereum node to connect to.
+   * @param ipfsUrls - Array of IPFS gateway URLs, tried in order with fallback.
+   * @param provider - The Ethereum provider to connect to.
    */
-  constructor(ipfsUrl: string, provider: JsonRpcApiProvider) {
+  constructor(ipfsUrls: string | string[], provider: JsonRpcApiProvider) {
     super(provider);
-    this.gatewayUrl = ipfsUrl.replace(/\/?$/, ""); // e.g. "https://gateway.pinata.cloud"
+    // Support both single URL (backward compatibility) and array of URLs
+    const urls = Array.isArray(ipfsUrls) ? ipfsUrls : [ipfsUrls];
+    this.gatewayUrls = urls.map(url => url.replace(/\/?$/, "")); // Remove trailing slash
   }
 
   /**
    * Changes the IPFS provider and target.
-   * @param ipfsUrl - The new URL of the IPFS network node.
+   * @param ipfsUrls - Array of IPFS gateway URLs or single URL string.
    */
-  public changeIpfsGatewayUrl(ipfsUrl: string): void {
-    this.gatewayUrl = ipfsUrl.replace(/\/?$/, "");
+  public changeIpfsGatewayUrl(ipfsUrls: string | string[]): void {
+    const urls = Array.isArray(ipfsUrls) ? ipfsUrls : [ipfsUrls];
+    this.gatewayUrls = urls.map(url => url.replace(/\/?$/, ""));
   }
 
   /**
@@ -373,6 +376,7 @@ export class DappnodeRepository extends ApmRepository {
 
   /**
    * Lists the contents of a directory pointed by the given hash.
+   * Tries multiple gateways in order until successful.
    * ipfs.dag.get => reutrns `Tsize`!
    *
    * TODO: research why the size is different, i.e for the hash QmWcJrobqhHF7GWpqEbxdv2cWCCXbACmq85Hh7aJ1eu8rn Tsize is 64461521 and size is 64446140
@@ -383,66 +387,101 @@ export class DappnodeRepository extends ApmRepository {
    */
   public async list(hash: string): Promise<IPFSEntry[]> {
     const cidStr = this.sanitizeIpfsPath(hash.toString());
-    const url = `${this.gatewayUrl}/ipfs/${cidStr}?format=dag-json`;
-    const res = await fetch(url, {
-      headers: { Accept: "application/vnd.ipld.dag-json" }
-    });
-    if (!res.ok) {
-      throw new Error(`Failed to list directory ${cidStr}: ${res.status} ${res.statusText}`);
-    }
+    const errors: string[] = [];
+
+    for (const gatewayUrl of this.gatewayUrls) {
+      try {
+        const url = `${gatewayUrl}/ipfs/${cidStr}?format=dag-json`;
+        const res = await fetch(url, {
+          headers: { Accept: "application/vnd.ipld.dag-json" }
+        });
+
+        if (!res.ok) {
+          errors.push(`Gateway ${gatewayUrl}: ${res.status} ${res.statusText}`);
+          continue;
+        }
 
-    const dagJson = (await res.json()) as {
-      Links?: Array<{
-        Name: string;
-        Hash: { "/": string };
-        Tsize: number;
-      }>;
-    };
+        const dagJson = (await res.json()) as {
+          Links?: Array<{
+            Name: string;
+            Hash: { "/": string };
+            Tsize: number;
+          }>;
+        };
+
+        if (!dagJson.Links) {
+          errors.push(`Gateway ${gatewayUrl}: Invalid IPFS directory CID ${cidStr}`);
+          continue;
+        }
 
-    if (!dagJson.Links) {
-      throw new Error(`Invalid IPFS directory CID ${cidStr}`);
+        // Success! Return the directory listing
+        return dagJson.Links.map((link) => ({
+          type: "file",
+          cid: CID.parse(this.sanitizeIpfsPath(link.Hash["/"])),
+          name: link.Name,
+          path: `${link.Hash["/"]}/${link.Name}`,
+          size: link.Tsize
+        }));
+
+      } catch (error) {
+        errors.push(`Gateway ${gatewayUrl}: ${error.message}`);
+      }
     }
 
-    return dagJson.Links.map((link) => ({
-      type: "file",
-      cid: CID.parse(this.sanitizeIpfsPath(link.Hash["/"])),
-      name: link.Name,
-      path: `${link.Hash["/"]}/${link.Name}`,
-      size: link.Tsize
-    }));
+    // If we get here, all gateways failed
+    throw new Error(`Failed to list directory from all gateways. Errors: ${errors.join("; ")}`);
   }
 
   /**
    * Gets the content from an IPFS gateway using the given hash and verifies its integrity.
+   * Tries multiple gateways in order until content is found or all gateways fail.
    * The content is returned as a CAR reader and the root CID.
    *
    * @param hash - The content identifier (CID) of the content to get and verify.
    * @returns The content as a CAR reader and the root CID.
-   * @throws Error when the root CID does not match the provided hash (content is untrusted).
+   * @throws Error when content is not found on any gateway or root CID does not match.
    */
   private async getAndVerifyContentFromGateway(hash: string): Promise<{
     carReader: CarReader;
     root: CID;
   }> {
-    // 1. Download the CAR
-    const url = `${this.gatewayUrl}/ipfs/${hash}?format=car`;
-    const res = await fetch(url, {
-      headers: { Accept: "application/vnd.ipld.car" }
-    });
-    if (!res.ok) throw new Error(`Gateway error: ${res.status} ${res.statusText}`);
+    const errors: string[] = [];
+
+    for (const gatewayUrl of this.gatewayUrls) {
+      try {
+        // Download the CAR directly - no need for HEAD check since we validate content anyway
+        const url = `${gatewayUrl}/ipfs/${hash}?format=car`;
+        const res = await fetch(url, {
+          headers: { Accept: "application/vnd.ipld.car" }
+        });
+
+        if (!res.ok) {
+          errors.push(`Gateway ${gatewayUrl}: ${res.status} ${res.statusText}`);
+          continue;
+        }
 
-    // 2. Parse into a CarReader
-    const bytes = new Uint8Array(await res.arrayBuffer());
-    const carReader = await CarReader.fromBytes(bytes);
+        // Parse into a CarReader
+        const bytes = new Uint8Array(await res.arrayBuffer());
+        const carReader = await CarReader.fromBytes(bytes);
 
-    // 3. Verify the root CID
-    const roots = await carReader.getRoots();
-    const root = roots[0];
-    if (roots.length !== 1 || root.toString() !== CID.parse(hash).toString()) {
-      throw new Error(`UNTRUSTED CONTENT: expected root ${hash}, got ${roots}`);
+        // Verify the root CID
+        const roots = await carReader.getRoots();
+        const root = roots[0];
+        if (roots.length !== 1 || root.toString() !== CID.parse(hash).toString()) {
+          errors.push(`Gateway ${gatewayUrl}: UNTRUSTED CONTENT: expected root ${hash}, got ${roots}`);
+          continue;
+        }
+
+        // Success! Return the verified content
+        return { carReader, root };
+
+      } catch (error) {
+        errors.push(`Gateway ${gatewayUrl}: ${error.message}`);
+      }
     }
 
-    return { carReader, root };
+    // If we get here, all gateways failed
+    throw new Error(`Failed to fetch content from all gateways. Errors: ${errors.join("; ")}`);
   }
 
   /**

packages/toolkit/test/repository/multiGateway.test.ts

@@ -0,0 +1,40 @@
+import { expect } from "chai";
+import { DappnodeRepository } from "../../src/index.js";
+import { JsonRpcApiProvider } from "ethers";
+
+describe("Multi-gateway IPFS", function () {
+  // Use null as provider since we're only testing the constructor and methods
+  const mockProvider = null as JsonRpcApiProvider;
+
+  it("Should support multiple gateways in constructor", function () {
+    const gateways = ["http://gateway1.example.com", "http://gateway2.example.com"];
+    const repo = new DappnodeRepository(gateways, mockProvider);
+
+    // The constructor should accept arrays
+    expect(repo).to.be.instanceOf(DappnodeRepository);
+  });
+
+  it("Should support single gateway for backward compatibility", function () {
+    const gateway = "http://gateway.example.com";
+    const repo = new DappnodeRepository(gateway, mockProvider);
+
+    // The constructor should still accept single strings
+    expect(repo).to.be.instanceOf(DappnodeRepository);
+  });
+
+  it("Should support changing gateways with arrays", function () {
+    const repo = new DappnodeRepository("http://initial.example.com", mockProvider);
+    const newGateways = ["http://new1.example.com", "http://new2.example.com"];
+
+    // Should not throw
+    repo.changeIpfsGatewayUrl(newGateways);
+  });
+
+  it("Should support changing gateways with strings", function () {
+    const repo = new DappnodeRepository(["http://initial.example.com"], mockProvider);
+    const newGateway = "http://new.example.com";
+
+    // Should not throw 
+    repo.changeIpfsGatewayUrl(newGateway);
+  });
+});