Skip to content
This repository has been archived by the owner on Feb 12, 2024. It is now read-only.

fix: cache preloaded CIDs #3363

Merged
merged 3 commits into from
Oct 30, 2020
Merged

fix: cache preloaded CIDs #3363

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
a190cdf
fix: cache preloaded CIDs
achingbrain Oct 30, 2020
8853543
chore: add test for caching of cids
achingbrain Oct 30, 2020
119027c
chore: apply suggestions from code review
achingbrain Oct 30, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
20 changes: 18 additions & 2 deletions packages/ipfs-core/src/preload.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,9 @@ const CID = require('cids')
const shuffle = require('array-shuffle')
const AbortController = require('native-abort-controller')
const preload = require('./runtime/preload-nodejs')
/** @type {typeof import('hashlru').default} */
// @ts-ignore - hashlru has incorrect typedefs
const hashlru = require('hashlru')

const log = Object.assign(
debug('ipfs:preload'),
Expand All @@ -14,12 +17,14 @@ const log = Object.assign(

/**
* @param {Object} [options]
* @param {boolean} [options.enabled]
* @param {string[]} [options.addresses]
* @param {boolean} [options.enabled = false] - Whether to preload anything
* @param {string[]} [options.addresses = []] - Which preload servers to use
* @param {number} [options.cache = 1000] - How many CIDs to cache
*/
const createPreloader = (options = {}) => {
options.enabled = Boolean(options.enabled)
options.addresses = options.addresses || []
options.cache = options.cache || 1000

if (!options.enabled || !options.addresses.length) {
log('preload disabled')
Expand All @@ -34,6 +39,9 @@ const createPreloader = (options = {}) => {
let requests = []
const apiUris = options.addresses.map(toUri)

// Avoid preloading the same CID over and over again
const cache = hashlru(options.cache)

/**
* @param {string|CID} path
* @returns {Promise<void>}
Expand All @@ -46,6 +54,14 @@ const createPreloader = (options = {}) => {
path = new CID(path).toString()
}

if (cache.has(path)) {
// we've preloaded this recently, don't preload it again
return
}

// make sure we don't preload this again any time soon
cache.set(path, true)

const fallbackApiUris = shuffle(apiUris)
let success = false
const now = Date.now()
Expand Down
39 changes: 27 additions & 12 deletions packages/ipfs-core/test/preload.spec.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,19 @@ describe('preload', () => {
after(() => cleanup())
afterEach(() => MockPreloadNode.clearPreloadCids())

it('should not preload content multiple times', async function () {
this.timeout(50 * 1000)
const { cid } = await ipfs.add(uint8ArrayFromString(nanoid()), { preload: false })

await all(ipfs.cat(cid))
await MockPreloadNode.waitForCids(cid)

// should not preload the second time
await MockPreloadNode.clearPreloadCids()
await all(ipfs.cat(cid))
await expect(MockPreloadNode.waitForCids(cid)).to.eventually.be.rejectedWith('Timed out waiting for CIDs to be preloaded')
})

it('should preload content added with add', async function () {
this.timeout(50 * 1000)
const res = await ipfs.add(uint8ArrayFromString(nanoid()))
Expand Down Expand Up @@ -112,7 +125,7 @@ describe('preload', () => {
}, {
path: 'dir0/file2',
content: uint8ArrayFromString(nanoid())
}], { wrapWithDirectory: true }))
}], { wrapWithDirectory: true, preload: false }))

const wrappingDir = res.find(file => file.path === '')
expect(wrappingDir).to.exist()
Expand Down Expand Up @@ -147,12 +160,12 @@ describe('preload', () => {

const [parent, link] = await Promise.all([createNode(), createNode()])

await MockPreloadNode.clearPreloadCids()
const cid = await ipfs.object.patch.addLink(parent.cid, {
Name: 'link',
Hash: link.cid,
Tsize: link.node.size
})

await MockPreloadNode.waitForCids(cid)
})

Expand All @@ -171,27 +184,31 @@ describe('preload', () => {
}]
})

await MockPreloadNode.clearPreloadCids()
const cid = await ipfs.object.patch.rmLink(parentCid, { name: 'link' })
await MockPreloadNode.waitForCids(cid)
})

it('should preload content added with object.patch.setData', async function () {
this.timeout(50 * 1000)
const originalCid = await ipfs.object.put({ Data: uint8ArrayFromString(nanoid()), Links: [] })
await MockPreloadNode.clearPreloadCids()
const patchedCid = await ipfs.object.patch.setData(originalCid, uint8ArrayFromString(nanoid()))
await MockPreloadNode.waitForCids(patchedCid)
})

it('should preload content added with object.patch.appendData', async function () {
this.timeout(50 * 1000)
const originalCid = await ipfs.object.put({ Data: uint8ArrayFromString(nanoid()), Links: [] })
await MockPreloadNode.clearPreloadCids()
const patchedCid = await ipfs.object.patch.appendData(originalCid, uint8ArrayFromString(nanoid()))
await MockPreloadNode.waitForCids(patchedCid)
})

it('should preload content retrieved with object.get', async function () {
this.timeout(50 * 1000)
const cid = await ipfs.object.new({ preload: false })
const cid = await ipfs.object.put({ Data: uint8ArrayFromString(nanoid()), Links: [] }, { preload: false })
await MockPreloadNode.clearPreloadCids()
await ipfs.object.get(cid)
await MockPreloadNode.waitForCids(cid)
})
Expand All @@ -205,13 +222,15 @@ describe('preload', () => {
it('should preload content retrieved with block.get', async function () {
this.timeout(50 * 1000)
const block = await ipfs.block.put(uint8ArrayFromString(nanoid()), { preload: false })
await MockPreloadNode.clearPreloadCids()
await ipfs.block.get(block.cid)
await MockPreloadNode.waitForCids(block.cid)
})

it('should preload content retrieved with block.stat', async function () {
this.timeout(50 * 1000)
const block = await ipfs.block.put(uint8ArrayFromString(nanoid()), { preload: false })
await MockPreloadNode.clearPreloadCids()
await ipfs.block.stat(block.cid)
await MockPreloadNode.waitForCids(block.cid)
})
Expand All @@ -228,39 +247,35 @@ describe('preload', () => {
const obj = { test: nanoid() }
const opts = { format: 'dag-cbor', hashAlg: 'sha2-256', preload: false }
const cid = await ipfs.dag.put(obj, opts)
await MockPreloadNode.clearPreloadCids()
await ipfs.dag.get(cid)
await MockPreloadNode.waitForCids(cid)
})

it('should preload content retrieved with files.ls', async () => {
const res = await ipfs.add({ path: `/t/${nanoid()}`, content: uint8ArrayFromString(nanoid()) })
const res = await ipfs.add({ path: `/t/${nanoid()}`, content: uint8ArrayFromString(nanoid()) }, { preload: false })
const dirCid = res.cid
await MockPreloadNode.waitForCids(dirCid)
await MockPreloadNode.clearPreloadCids()
await all(ipfs.files.ls(`/ipfs/${dirCid}`))
await MockPreloadNode.waitForCids(`/ipfs/${dirCid}`)
})

it('should preload content retrieved with files.ls by CID', async () => {
const res = await ipfs.add({ path: `/t/${nanoid()}`, content: uint8ArrayFromString(nanoid()) })
const res = await ipfs.add({ path: `/t/${nanoid()}`, content: uint8ArrayFromString(nanoid()) }, { preload: false })
const dirCid = res.cid
await MockPreloadNode.waitForCids(dirCid)
await MockPreloadNode.clearPreloadCids()
await all(ipfs.files.ls(dirCid))
await MockPreloadNode.waitForCids(dirCid)
})

it('should preload content retrieved with files.read', async () => {
const { cid } = await ipfs.add(uint8ArrayFromString(nanoid()))
await MockPreloadNode.waitForCids(cid)
const { cid } = await ipfs.add(uint8ArrayFromString(nanoid()), { preload: false })
await MockPreloadNode.clearPreloadCids()
await ipfs.files.read(`/ipfs/${cid}`)
await MockPreloadNode.waitForCids(`/ipfs/${cid}`)
})

it('should preload content retrieved with files.stat', async () => {
const { cid: fileCid } = await ipfs.add(uint8ArrayFromString(nanoid()))
await MockPreloadNode.waitForCids(fileCid)
const { cid: fileCid } = await ipfs.add(uint8ArrayFromString(nanoid()), { preload: false })
await MockPreloadNode.clearPreloadCids()
await ipfs.files.stat(`/ipfs/${fileCid}`)
await MockPreloadNode.waitForCids(`/ipfs/${fileCid}`)
Expand Down