fix(dag): replace custom dag walk with multiformats/traversal (#3950)

Ref: multiformats/js-multiformats#127

Co-authored-by: achingbrain <alex@achingbrain.net>

Author: rvagg

packages/interface-ipfs-core/package.json

@@ -94,7 +94,7 @@
     "libp2p-websockets": "^0.16.2",
     "merge-options": "^3.0.4",
     "multiaddr": "^10.0.0",
-    "multiformats": "^9.4.13",
+    "multiformats": "^9.5.1",
     "nanoid": "^3.1.23",
     "p-map": "^4.0.0",
     "p-retry": "^4.5.0",

packages/ipfs-cli/package.json

@@ -86,7 +86,7 @@
     "mafmt": "^10.0.0",
     "multiaddr": "^10.0.0",
     "multiaddr-to-uri": "^8.0.0",
-    "multiformats": "^9.4.13",
+    "multiformats": "^9.5.1",
     "parse-duration": "^1.0.0",
     "pretty-bytes": "^5.4.1",
     "progress": "^2.0.3",

packages/ipfs-core-types/package.json

@@ -45,7 +45,7 @@
   "dependencies": {
     "interface-datastore": "^6.0.2",
     "multiaddr": "^10.0.0",
-    "multiformats": "^9.4.13"
+    "multiformats": "^9.5.1"
   },
   "devDependencies": {
     "aegir": "^36.0.1",

packages/ipfs-core-utils/package.json

@@ -129,7 +129,7 @@
     "merge-options": "^3.0.4",
     "multiaddr": "^10.0.0",
     "multiaddr-to-uri": "^8.0.0",
-    "multiformats": "^9.4.13",
+    "multiformats": "^9.5.1",
     "nanoid": "^3.1.23",
     "parse-duration": "^1.0.0",
     "timeout-abort-controller": "^3.0.0",

packages/ipfs-core/package.json

@@ -124,7 +124,7 @@
     "mortice": "^2.0.0",
     "multiaddr": "^10.0.0",
     "multiaddr-to-uri": "^8.0.0",
-    "multiformats": "^9.4.13",
+    "multiformats": "^9.5.1",
     "pako": "^1.0.2",
     "parse-duration": "^1.0.0",
     "peer-id": "^0.16.0",

packages/ipfs-core/src/components/dag/export.js

@@ -1,11 +1,11 @@
 import { CID } from 'multiformats/cid'
 import { createUnsafe } from 'multiformats/block'
-import { base58btc } from 'multiformats/bases/base58'
 import { CarWriter } from '@ipld/car/writer'
 import { withTimeoutOption } from 'ipfs-core-utils/with-timeout-option'
 import debug from 'debug'
 import * as raw from 'multiformats/codecs/raw'
 import * as json from 'multiformats/codecs/json'
+import { walk } from 'multiformats/traversal'
 
 const log = debug('ipfs:components:dag:import')
 
@@ -23,6 +23,11 @@ const NO_LINKS_CODECS = [
  * @typedef {import('ipfs-core-types/src/utils').AbortOptions} AbortOptions
  */
 
+/**
+ * @template T
+ * @typedef {import('multiformats/block').Block<T>} Block
+ */
+
 /**
  * @param {Object} config
  * @param {IPFSRepo} config.repo
@@ -53,12 +58,11 @@ export function createExport ({ repo, preload, codecs }) {
     let err = null
     ;(async () => {
       try {
-        await traverseWrite(
-          repo,
-          { signal: options.signal, timeout: options.timeout },
-          cid,
-          writer,
-          codecs)
+        const load = makeLoader(repo, writer, {
+          signal: options.signal,
+          timeout: options.timeout
+        }, codecs)
+        await walk({ cid, load })
       } catch (/** @type {any} */ e) {
         err = e
       } finally {
@@ -81,52 +85,30 @@ export function createExport ({ repo, preload, codecs }) {
 }
 
 /**
+ * @template T
  * @param {IPFSRepo} repo
- * @param {AbortOptions} options
- * @param {CID} cid
  * @param {BlockWriter} writer
+ * @param {AbortOptions} options
  * @param {import('ipfs-core-utils/multicodecs').Multicodecs} codecs
- * @param {Set<string>} seen
- * @returns {Promise<void>}
+ * @returns {(cid:CID)=>Promise<Block<T>|null>}
  */
-async function traverseWrite (repo, options, cid, writer, codecs, seen = new Set()) {
-  const b58Cid = cid.toString(base58btc)
-  if (seen.has(b58Cid)) {
-    return
-  }
+function makeLoader (repo, writer, options, codecs) {
+  return async (cid) => {
+    const codec = await codecs.getCodec(cid.code)
 
-  const block = await getBlock(repo, options, cid, codecs)
+    if (!codec) {
+      throw new Error(`Can't decode links in block with codec 0x${cid.code.toString(16)} to form complete DAG`)
+    }
 
-  log(`Adding block ${cid} to car`)
-  await writer.put(block)
-  seen.add(b58Cid)
+    const bytes = await repo.blocks.get(cid, options)
 
-  // recursive traversal of all links
-  for (const link of block.links) {
-    await traverseWrite(repo, options, link, writer, codecs, seen)
-  }
-}
+    log(`Adding block ${cid} to car`)
+    await writer.put({ cid, bytes })
 
-/**
- * @param {IPFSRepo} repo
- * @param {AbortOptions} options
- * @param {CID} cid
- * @param {import('ipfs-core-utils/multicodecs').Multicodecs} codecs
- * @returns {Promise<{cid:CID, bytes:Uint8Array, links:CID[]}>}
- */
-async function getBlock (repo, options, cid, codecs) {
-  const bytes = await repo.blocks.get(cid, options)
-
-  /** @type {CID[]} */
-  let links = []
-  const codec = await codecs.getCodec(cid.code)
-
-  if (codec) {
-    const block = createUnsafe({ bytes, cid, codec })
-    links = [...block.links()].map((l) => l[1])
-  } else if (!NO_LINKS_CODECS.includes(cid.code)) {
-    throw new Error(`Can't decode links in block with codec 0x${cid.code.toString(16)} to form complete DAG`)
-  }
+    if (NO_LINKS_CODECS.includes(cid.code)) {
+      return null // skip this block, no need to look inside
+    }
 
-  return { cid, bytes, links }
+    return createUnsafe({ bytes, cid, codec })
+  }
 }

packages/ipfs-grpc-client/package.json

@@ -63,7 +63,7 @@
     "it-first": "^1.0.4",
     "it-pushable": "^1.4.2",
     "multiaddr": "^10.0.0",
-    "multiformats": "^9.4.13",
+    "multiformats": "^9.5.1",
     "p-defer": "^3.0.0",
     "protobufjs": "^6.10.2",
     "wherearewe": "^1.0.0",

packages/ipfs-http-client/package.json

@@ -68,7 +68,7 @@
     "it-last": "^1.0.4",
     "merge-options": "^3.0.4",
     "multiaddr": "^10.0.0",
-    "multiformats": "^9.4.13",
+    "multiformats": "^9.5.1",
     "parse-duration": "^1.0.0",
     "stream-to-it": "^0.2.2",
     "uint8arrays": "^3.0.0"

packages/ipfs-http-gateway/package.json

@@ -68,7 +68,7 @@
     "it-last": "^1.0.4",
     "it-to-stream": "^1.0.0",
     "joi": "^17.2.1",
-    "multiformats": "^9.4.13",
+    "multiformats": "^9.5.1",
     "uint8arrays": "^3.0.0",
     "uri-to-multiaddr": "^6.0.0"
   },

packages/ipfs-http-server/package.json

@@ -71,7 +71,7 @@
     "joi": "^17.2.1",
     "just-safe-set": "^2.2.1",
     "multiaddr": "^10.0.0",
-    "multiformats": "^9.4.13",
+    "multiformats": "^9.5.1",
     "parse-duration": "^1.0.0",
     "stream-to-it": "^0.2.2",
     "timeout-abort-controller": "^3.0.0",

packages/ipfs-message-port-client/package.json

@@ -50,7 +50,7 @@
     "ipfs-message-port-protocol": "^0.11.0",
     "ipfs-unixfs": "^6.0.3",
     "it-peekable": "^1.0.2",
-    "multiformats": "^9.4.13"
+    "multiformats": "^9.5.1"
   },
   "devDependencies": {
     "aegir": "^36.0.1",

packages/ipfs-message-port-protocol/package.json

@@ -75,7 +75,7 @@
   },
   "dependencies": {
     "ipfs-core-types": "^0.10.0",
-    "multiformats": "^9.4.13"
+    "multiformats": "^9.5.1"
   },
   "devDependencies": {
     "aegir": "^36.0.1",

packages/ipfs-message-port-server/package.json

@@ -71,7 +71,7 @@
     "ipfs-core-types": "^0.10.0",
     "ipfs-message-port-protocol": "^0.11.0",
     "it-all": "^1.0.4",
-    "multiformats": "^9.4.13"
+    "multiformats": "^9.5.1"
   },
   "devDependencies": {
     "aegir": "^36.0.1",