Skip to content
This repository was archived by the owner on Feb 12, 2024. It is now read-only.

Commit 226a138

Browse files
achingbrainachingbrain
authored
chore: upgrade ipfs-unixfs-importer to use block api only (#3003)
Removes use of ipld by the importer, instead using the block api directly. This allows use by implementations that have only completed low-level APIs and also makes importing ever so slightly faster by not serializing nodes twice while adding them to the block store.
1 parent 553a65b commit 226a138

File tree

11 files changed

+57
-40
lines changed

11 files changed

+57
-40
lines changed

‎packages/interface-ipfs-core/src/files/write.js

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -780,7 +780,7 @@ module.exports = (common, options) => {
780780
const stats = await ipfs.files.stat(dirPath)
781781
const updatedDirCid = stats.cid
782782

783-
expect(updatedDirCid.toString()).to.deep.equal('QmcGTKoaZeMxVenyxnkP2riibE8vSEPobkN1oxvcEZpBW5')
783+
expect(updatedDirCid.toString()).to.equal('QmcGTKoaZeMxVenyxnkP2riibE8vSEPobkN1oxvcEZpBW5')
784784
})
785785

786786
it('results in the same hash as a sharded directory created by the importer when adding a file to a subshard', async function () {

‎packages/ipfs/package.json

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -104,8 +104,8 @@
104104
"ipfs-http-response": "^0.5.0",
105105
"ipfs-repo": "^1.0.1",
106106
"ipfs-unixfs": "^1.0.1",
107-
"ipfs-unixfs-exporter": "^1.0.3",
108-
"ipfs-unixfs-importer": "^1.0.3",
107+
"ipfs-unixfs-exporter": "^2.0.0",
108+
"ipfs-unixfs-importer": "^2.0.0",
109109
"ipfs-utils": "^2.2.0",
110110
"ipld": "^0.25.0",
111111
"ipld-bitcoin": "^0.3.0",

‎packages/ipfs/src/core/components/add/index.js

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@ const normaliseAddInput = require('ipfs-core-utils/src/files/normalise-input')
55
const { parseChunkerString } = require('./utils')
66
const pipe = require('it-pipe')
77

8-
module.exports = ({ ipld, gcLock, preload, pin, options: constructorOptions }) => {
8+
module.exports = ({ block, gcLock, preload, pin, options: constructorOptions }) => {
99
const isShardingEnabled = constructorOptions.EXPERIMENTAL && constructorOptions.EXPERIMENTAL.sharding
1010
return async function * add (source, options) {
1111
options = options || {}
@@ -40,7 +40,10 @@ module.exports = ({ ipld, gcLock, preload, pin, options: constructorOptions }) =
4040

4141
const iterator = pipe(
4242
normaliseAddInput(source),
43-
source => importer(source, ipld, opts),
43+
source => importer(source, block, {
44+
...opts,
45+
pin: false
46+
}),
4447
transformFile(opts),
4548
preloadFile(preload, opts),
4649
pinFile(pin, opts)

‎packages/ipfs/src/core/components/files/chmod.js

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -192,12 +192,13 @@ module.exports = (context) => {
192192
}
193193
}
194194
},
195-
(source) => importer(source, context.ipld, {
195+
(source) => importer(source, context.block, {
196196
...options,
197-
dagBuilder: async function * (source, ipld, options) {
197+
pin: false,
198+
dagBuilder: async function * (source, block, options) {
198199
for await (const entry of source) {
199200
yield async function () {
200-
const cid = await persist(entry.content, ipld, options)
201+
const cid = await persist(entry.content.serialize(), block, options)
201202

202203
return {
203204
cid,

‎packages/ipfs/src/core/components/files/index.js

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -76,9 +76,10 @@ function createMfs (options) {
7676
return mfs
7777
}
7878

79-
module.exports = ({ ipld, blockService, repo, preload, options: constructorOptions }) => {
79+
module.exports = ({ ipld, block, blockService, repo, preload, options: constructorOptions }) => {
8080
const methods = createMfs({
8181
ipld,
82+
block,
8283
blocks: blockService,
8384
datastore: repo.root,
8485
repoOwner: constructorOptions.repoOwner

‎packages/ipfs/src/core/components/files/utils/add-link.js

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -124,7 +124,7 @@ const addToShardedDirectory = async (context, options) => {
124124
shard, path
125125
} = await addFileToShardedDirectory(context, options)
126126

127-
const result = await last(shard.flush('', context.ipld))
127+
const result = await last(shard.flush('', context.block))
128128
const node = await context.ipld.get(result.cid)
129129

130130
// we have written out the shard, but only one sub-shard will have been written so replace it in the original shard
@@ -203,7 +203,7 @@ const addFileToShardedDirectory = async (context, options) => {
203203

204204
if (link.Name.length > 2) {
205205
// another file had the same prefix, will be replaced with a subshard
206-
log(`Link ${link.Name} will be replaced with a subshard`)
206+
log(`Link ${link.Name} ${link.Hash} will be replaced with a subshard`)
207207
index = path.length
208208

209209
break
@@ -231,7 +231,7 @@ const addFileToShardedDirectory = async (context, options) => {
231231

232232
const nextSegment = path[index]
233233

234-
// add next level's worth of links to bucket
234+
// add next levels worth of links to bucket
235235
await addLinksToHamtBucket(subShard.Links, nextSegment.bucket, rootBucket)
236236

237237
nextSegment.node = subShard

‎packages/ipfs/src/core/components/files/utils/hamt-utils.js

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -69,7 +69,7 @@ const addLinksToHamtBucket = async (links, bucket, rootBucket) => {
6969
}
7070

7171
return (rootBucket || bucket).put(link.Name.substring(2), {
72-
size: link.TSize,
72+
size: link.Tsize,
7373
cid: link.Hash
7474
})
7575
})
@@ -195,7 +195,7 @@ const createShard = async (context, contents, options) => {
195195
})
196196
}
197197

198-
return last(shard.flush('', context.ipld, null))
198+
return last(shard.flush('', context.block, null))
199199
}
200200

201201
module.exports = {

‎packages/ipfs/src/core/components/files/write.js

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -204,14 +204,15 @@ const write = async (context, source, destination, options) => {
204204
// persist mode & mtime if set previously
205205
mode,
206206
mtime
207-
}], context.ipld, {
207+
}], context.block, {
208208
progress: options.progress,
209209
hashAlg: options.hashAlg,
210210
cidVersion: options.cidVersion,
211211
strategy: options.strategy,
212212
rawLeaves: options.rawLeaves,
213213
reduceSingleLeafToSelf: options.reduceSingleLeafToSelf,
214-
leafType: options.leafType
214+
leafType: options.leafType,
215+
pin: false
215216
}))
216217

217218
log(`Wrote ${result.cid}`)

‎packages/ipfs/src/core/components/init.js

Lines changed: 12 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -121,7 +121,14 @@ module.exports = ({
121121
// FIXME: resolve this circular dependency
122122
dag.put = Components.dag.put({ ipld, pin, gcLock, preload })
123123

124-
const add = Components.add({ ipld, preload, pin, gcLock, options: constructorOptions })
124+
const block = {
125+
get: Components.block.get({ blockService, preload }),
126+
put: Components.block.put({ blockService, gcLock, preload }),
127+
rm: Components.block.rm({ blockService, gcLock, pinManager }),
128+
stat: Components.block.stat({ blockService, preload })
129+
}
130+
131+
const add = Components.add({ block, preload, pin, gcLock, options: constructorOptions })
125132

126133
if (!isInitialized && !options.emptyRepo) {
127134
// add empty unixfs dir object (go-ipfs assumes this exists)
@@ -142,6 +149,7 @@ module.exports = ({
142149
add,
143150
apiManager,
144151
constructorOptions,
152+
block,
145153
blockService,
146154
dag,
147155
gcLock,
@@ -286,6 +294,7 @@ function createApi ({
286294
add,
287295
apiManager,
288296
constructorOptions,
297+
block,
289298
blockService,
290299
dag,
291300
gcLock,
@@ -320,17 +329,12 @@ function createApi ({
320329
list: Components.bootstrap.list({ repo }),
321330
rm: Components.bootstrap.rm({ repo })
322331
},
323-
block: {
324-
get: Components.block.get({ blockService, preload }),
325-
put: Components.block.put({ blockService, gcLock, preload }),
326-
rm: Components.block.rm({ blockService, gcLock, pinManager }),
327-
stat: Components.block.stat({ blockService, preload })
328-
},
332+
block,
329333
cat: Components.cat({ ipld, preload }),
330334
config: Components.config({ repo }),
331335
dag,
332336
dns: Components.dns(),
333-
files: Components.files({ ipld, blockService, repo, preload, options: constructorOptions }),
337+
files: Components.files({ ipld, block, blockService, repo, preload, options: constructorOptions }),
334338
get: Components.get({ ipld, preload }),
335339
id: Components.id({ peerInfo }),
336340
init: async () => { throw new AlreadyInitializedError() }, // eslint-disable-line require-await

‎packages/ipfs/src/core/components/start.js

Lines changed: 12 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -71,7 +71,14 @@ module.exports = ({
7171

7272
blockService.setExchange(bitswap)
7373

74-
const files = Components.files({ ipld, blockService, repo, preload, options: constructorOptions })
74+
const block = {
75+
get: Components.block.get({ blockService, preload }),
76+
put: Components.block.put({ blockService, gcLock, preload }),
77+
rm: Components.block.rm({ blockService, gcLock, pinManager }),
78+
stat: Components.block.stat({ blockService, preload })
79+
}
80+
81+
const files = Components.files({ ipld, block, blockService, repo, preload, options: constructorOptions })
7582
const mfsPreload = createMfsPreload({ files, preload, options: constructorOptions.preload })
7683

7784
await Promise.all([
@@ -83,6 +90,7 @@ module.exports = ({
8390
const api = createApi({
8491
apiManager,
8592
bitswap,
93+
block,
8694
blockService,
8795
config,
8896
constructorOptions,
@@ -115,6 +123,7 @@ module.exports = ({
115123
function createApi ({
116124
apiManager,
117125
bitswap,
126+
block,
118127
blockService,
119128
config,
120129
constructorOptions,
@@ -158,7 +167,7 @@ function createApi ({
158167
}
159168
// FIXME: resolve this circular dependency
160169
dag.put = Components.dag.put({ ipld, pin, gcLock, preload })
161-
const add = Components.add({ ipld, preload, pin, gcLock, options: constructorOptions })
170+
const add = Components.add({ block, preload, pin, gcLock, options: constructorOptions })
162171
const isOnline = Components.isOnline({ libp2p })
163172

164173
const dhtNotEnabled = async () => { // eslint-disable-line require-await
@@ -209,12 +218,7 @@ function createApi ({
209218
unwant: Components.bitswap.unwant({ bitswap }),
210219
wantlist: Components.bitswap.wantlist({ bitswap })
211220
},
212-
block: {
213-
get: Components.block.get({ blockService, preload }),
214-
put: Components.block.put({ blockService, gcLock, preload }),
215-
rm: Components.block.rm({ blockService, gcLock, pinManager }),
216-
stat: Components.block.stat({ blockService, preload })
217-
},
221+
block,
218222
bootstrap: {
219223
add: Components.bootstrap.add({ repo }),
220224
list: Components.bootstrap.list({ repo }),

‎packages/ipfs/src/core/components/stop.js

Lines changed: 11 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -106,7 +106,15 @@ function createApi ({
106106
}
107107
// FIXME: resolve this circular dependency
108108
dag.put = Components.dag.put({ ipld, pin, gcLock, preload })
109-
const add = Components.add({ ipld, preload, pin, gcLock, options: constructorOptions })
109+
110+
const block = {
111+
get: Components.block.get({ blockService, preload }),
112+
put: Components.block.put({ blockService, gcLock, preload }),
113+
rm: Components.block.rm({ blockService, gcLock, pinManager }),
114+
stat: Components.block.stat({ blockService, preload })
115+
}
116+
117+
const add = Components.add({ block, preload, pin, gcLock, options: constructorOptions })
110118
const resolve = Components.resolve({ ipld })
111119
const refs = Components.refs({ ipld, resolve, preload })
112120
refs.local = Components.refs.local({ repo })
@@ -122,12 +130,7 @@ function createApi ({
122130
unwant: notStarted,
123131
wantlist: notStarted
124132
},
125-
block: {
126-
get: Components.block.get({ blockService, preload }),
127-
put: Components.block.put({ blockService, gcLock, preload }),
128-
rm: Components.block.rm({ blockService, gcLock, pinManager }),
129-
stat: Components.block.stat({ blockService, preload })
130-
},
133+
block,
131134
bootstrap: {
132135
add: Components.bootstrap.add({ repo }),
133136
list: Components.bootstrap.list({ repo }),
@@ -137,7 +140,7 @@ function createApi ({
137140
config: Components.config({ repo }),
138141
dag,
139142
dns: Components.dns(),
140-
files: Components.files({ ipld, blockService, repo, preload, options: constructorOptions }),
143+
files: Components.files({ ipld, block, blockService, repo, preload, options: constructorOptions }),
141144
get: Components.get({ ipld, preload }),
142145
id: Components.id({ peerInfo }),
143146
init: async () => { // eslint-disable-line require-await

0 commit comments

Comments
 (0)