Skip to content
This repository was archived by the owner on Feb 12, 2024. It is now read-only.

Commit 58aee9e

Browse files
rvaggrvagg
committed
chore(dag): replace custom dag walk with multiformats/traversal
Ref: multiformats/js-multiformats#127
1 parent 6178708 commit 58aee9e

File tree

13 files changed

+39
-61
lines changed

13 files changed

+39
-61
lines changed

‎packages/interface-ipfs-core/package.json

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -93,7 +93,7 @@
9393
"libp2p-crypto": "^0.19.7",
9494
"libp2p-websockets": "^0.16.2",
9595
"multiaddr": "^10.0.0",
96-
"multiformats": "^9.4.13",
96+
"multiformats": "^9.5.0",
9797
"nanoid": "^3.1.23",
9898
"native-abort-controller": "^1.0.3",
9999
"p-map": "^4.0.0",

‎packages/ipfs-cli/package.json

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -86,7 +86,7 @@
8686
"mafmt": "^10.0.0",
8787
"multiaddr": "^10.0.0",
8888
"multiaddr-to-uri": "^8.0.0",
89-
"multiformats": "^9.4.13",
89+
"multiformats": "^9.5.0",
9090
"parse-duration": "^1.0.0",
9191
"pretty-bytes": "^5.4.1",
9292
"progress": "^2.0.3",

‎packages/ipfs-core-types/package.json

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -46,7 +46,7 @@
4646
"dependencies": {
4747
"interface-datastore": "^6.0.2",
4848
"multiaddr": "^10.0.0",
49-
"multiformats": "^9.4.13"
49+
"multiformats": "^9.5.0"
5050
},
5151
"devDependencies": {
5252
"aegir": "^36.0.1",

‎packages/ipfs-core-utils/package.json

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -127,7 +127,7 @@
127127
"merge-options": "^3.0.4",
128128
"multiaddr": "^10.0.0",
129129
"multiaddr-to-uri": "^8.0.0",
130-
"multiformats": "^9.4.13",
130+
"multiformats": "^9.5.0",
131131
"nanoid": "^3.1.23",
132132
"parse-duration": "^1.0.0",
133133
"timeout-abort-controller": "^1.1.1",

‎packages/ipfs-core/package.json

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -121,7 +121,7 @@
121121
"mortice": "^2.0.0",
122122
"multiaddr": "^10.0.0",
123123
"multiaddr-to-uri": "^8.0.0",
124-
"multiformats": "^9.4.13",
124+
"multiformats": "^9.5.0",
125125
"native-abort-controller": "^1.0.3",
126126
"pako": "^1.0.2",
127127
"parse-duration": "^1.0.0",

‎packages/ipfs-core/src/components/dag/export.js

Lines changed: 27 additions & 49 deletions
Original file line numberDiff line numberDiff line change
@@ -1,11 +1,11 @@
11
import { CID } from 'multiformats/cid'
22
import { createUnsafe } from 'multiformats/block'
3-
import { base58btc } from 'multiformats/bases/base58'
43
import { CarWriter } from '@ipld/car/writer'
54
import { withTimeoutOption } from 'ipfs-core-utils/with-timeout-option'
65
import debug from 'debug'
76
import * as raw from 'multiformats/codecs/raw'
87
import * as json from 'multiformats/codecs/json'
8+
import { walk } from 'multiformats/traversal'
99

1010
const log = debug('ipfs:components:dag:import')
1111

@@ -23,6 +23,11 @@ const NO_LINKS_CODECS = [
2323
* @typedef {import('ipfs-core-types/src/utils').AbortOptions} AbortOptions
2424
*/
2525

26+
/**
27+
* @template T
28+
* @typedef {import('multiformats/block').Block<T>} Block
29+
*/
30+
2631
/**
2732
* @param {Object} config
2833
* @param {IPFSRepo} config.repo
@@ -53,13 +58,9 @@ export function createExport ({ repo, preload, codecs }) {
5358
let err = null
5459
;(async () => {
5560
try {
56-
await traverseWrite(
57-
repo,
58-
{ signal: options.signal, timeout: options.timeout },
59-
cid,
60-
writer,
61-
codecs)
62-
writer.close()
61+
const load = makeLoader(repo, writer, { signal: options.signal, timeout: options.timeout }, codecs)
62+
await walk({ cid, load })
63+
await writer.close()
6364
} catch (/** @type {any} */ e) {
6465
err = e
6566
}
@@ -80,52 +81,29 @@ export function createExport ({ repo, preload, codecs }) {
8081
}
8182

8283
/**
84+
* @template T
8385
* @param {IPFSRepo} repo
84-
* @param {AbortOptions} options
85-
* @param {CID} cid
8686
* @param {BlockWriter} writer
87-
* @param {import('ipfs-core-utils/multicodecs').Multicodecs} codecs
88-
* @param {Set<string>} seen
89-
* @returns {Promise<void>}
90-
*/
91-
async function traverseWrite (repo, options, cid, writer, codecs, seen = new Set()) {
92-
const b58Cid = cid.toString(base58btc)
93-
if (seen.has(b58Cid)) {
94-
return
95-
}
96-
97-
const block = await getBlock(repo, options, cid, codecs)
98-
99-
log(`Adding block ${cid} to car`)
100-
await writer.put(block)
101-
seen.add(b58Cid)
102-
103-
// recursive traversal of all links
104-
for (const link of block.links) {
105-
await traverseWrite(repo, options, link, writer, codecs, seen)
106-
}
107-
}
108-
109-
/**
110-
* @param {IPFSRepo} repo
11187
* @param {AbortOptions} options
112-
* @param {CID} cid
11388
* @param {import('ipfs-core-utils/multicodecs').Multicodecs} codecs
114-
* @returns {Promise<{cid:CID, bytes:Uint8Array, links:CID[]}>}
89+
* @returns {(cid:CID)=>Promise<Block<T>>}
11590
*/
116-
async function getBlock (repo, options, cid, codecs) {
117-
const bytes = await repo.blocks.get(cid, options)
118-
119-
/** @type {CID[]} */
120-
let links = []
121-
const codec = await codecs.getCodec(cid.code)
91+
function makeLoader (repo, writer, options, codecs) {
92+
return async (cid) => {
93+
const bytes = await repo.blocks.get(cid, options)
94+
const codec = await codecs.getCodec(cid.code)
95+
96+
let block
97+
if (codec) {
98+
block = createUnsafe({ bytes, cid, codec })
99+
} else if (NO_LINKS_CODECS.includes(cid.code)) {
100+
return null // skip this block
101+
} else {
102+
throw new Error(`Can't decode links in block with codec 0x${cid.code.toString(16)} to form complete DAG`)
103+
}
122104

123-
if (codec) {
124-
const block = createUnsafe({ bytes, cid, codec })
125-
links = [...block.links()].map((l) => l[1])
126-
} else if (!NO_LINKS_CODECS.includes(cid.code)) {
127-
throw new Error(`Can't decode links in block with codec 0x${cid.code.toString(16)} to form complete DAG`)
105+
log(`Adding block ${cid} to car`)
106+
await writer.put(block)
107+
return block
128108
}
129-
130-
return { cid, bytes, links }
131109
}

‎packages/ipfs-grpc-client/package.json

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -61,7 +61,7 @@
6161
"it-first": "^1.0.4",
6262
"it-pushable": "^1.4.2",
6363
"multiaddr": "^10.0.0",
64-
"multiformats": "^9.4.13",
64+
"multiformats": "^9.5.0",
6565
"p-defer": "^3.0.0",
6666
"protobufjs": "^6.10.2",
6767
"wherearewe": "^1.0.0",

‎packages/ipfs-http-client/package.json

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -67,7 +67,7 @@
6767
"it-last": "^1.0.4",
6868
"merge-options": "^3.0.4",
6969
"multiaddr": "^10.0.0",
70-
"multiformats": "^9.4.13",
70+
"multiformats": "^9.5.0",
7171
"native-abort-controller": "^1.0.3",
7272
"parse-duration": "^1.0.0",
7373
"stream-to-it": "^0.2.2",

‎packages/ipfs-http-gateway/package.json

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -69,7 +69,7 @@
6969
"it-last": "^1.0.4",
7070
"it-to-stream": "^1.0.0",
7171
"joi": "^17.2.1",
72-
"multiformats": "^9.4.13",
72+
"multiformats": "^9.5.0",
7373
"uint8arrays": "^3.0.0",
7474
"uri-to-multiaddr": "^6.0.0"
7575
},

‎packages/ipfs-http-server/package.json

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -72,7 +72,7 @@
7272
"joi": "^17.2.1",
7373
"just-safe-set": "^2.2.1",
7474
"multiaddr": "^10.0.0",
75-
"multiformats": "^9.4.13",
75+
"multiformats": "^9.5.0",
7676
"native-abort-controller": "^1.0.3",
7777
"parse-duration": "^1.0.0",
7878
"stream-to-it": "^0.2.2",

‎packages/ipfs-message-port-client/package.json

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -51,7 +51,7 @@
5151
"ipfs-message-port-protocol": "^0.10.4",
5252
"ipfs-unixfs": "^6.0.3",
5353
"it-peekable": "^1.0.2",
54-
"multiformats": "^9.4.13"
54+
"multiformats": "^9.5.0"
5555
},
5656
"devDependencies": {
5757
"aegir": "^36.0.1",

‎packages/ipfs-message-port-protocol/package.json

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -74,7 +74,7 @@
7474
},
7575
"dependencies": {
7676
"ipfs-core-types": "^0.8.4",
77-
"multiformats": "^9.4.13"
77+
"multiformats": "^9.5.0"
7878
},
7979
"devDependencies": {
8080
"aegir": "^36.0.1",

‎packages/ipfs-message-port-server/package.json

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -70,7 +70,7 @@
7070
"ipfs-core-types": "^0.8.4",
7171
"ipfs-message-port-protocol": "^0.10.4",
7272
"it-all": "^1.0.4",
73-
"multiformats": "^9.4.13"
73+
"multiformats": "^9.5.0"
7474
},
7575
"devDependencies": {
7676
"aegir": "^36.0.1",

0 commit comments

Comments
 (0)