Skip to content
This repository has been archived by the owner on Feb 12, 2024. It is now read-only.

Commit

Permalink
fix: cache preloaded CIDs (#3363)
Browse files Browse the repository at this point in the history 
We 'preload' most CIDs we interact with on the network. In some cases
this can mean preloading the same CID over and over again which is not
necessary.

This PR adds a LRU cache to the preloader with a default size of 1000.
The cache is used to avoid re-preloading the same CID over and over again
until it drops out of the cache.  We use a cache that will evict CIDs
over time to have some sort of upper bound on memory usage.

Fixes #3307

Co-authored-by: Vasco Santos <vasco.santos@moxy.studio>
  • Loading branch information
@achingbrain @vasco-santos
achingbrain and vasco-santos authored Oct 30, 2020
1 parent a542882 commit b5ea76a
Show file tree
Hide file tree
Showing 2 changed files with 45 additions and 14 deletions.
20 changes: 18 additions & 2 deletions packages/ipfs-core/src/preload.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,9 @@ const CID = require('cids')
const shuffle = require('array-shuffle')
const AbortController = require('native-abort-controller')
const preload = require('./runtime/preload-nodejs')
/** @type {typeof import('hashlru').default} */
// @ts-ignore - hashlru has incorrect typedefs
const hashlru = require('hashlru')

const log = Object.assign(
debug('ipfs:preload'),
Expand All @@ -14,12 +17,14 @@ const log = Object.assign(

/**
* @param {Object} [options]
* @param {boolean} [options.enabled]
* @param {string[]} [options.addresses]
* @param {boolean} [options.enabled = false] - Whether to preload anything
* @param {string[]} [options.addresses = []] - Which preload servers to use
* @param {number} [options.cache = 1000] - How many CIDs to cache
*/
const createPreloader = (options = {}) => {
options.enabled = Boolean(options.enabled)
options.addresses = options.addresses || []
options.cache = options.cache || 1000

if (!options.enabled || !options.addresses.length) {
log('preload disabled')
Expand All @@ -34,6 +39,9 @@ const createPreloader = (options = {}) => {
let requests = []
const apiUris = options.addresses.map(toUri)

// Avoid preloading the same CID over and over again
const cache = hashlru(options.cache)

/**
* @param {string|CID} path
* @returns {Promise<void>}
Expand All @@ -46,6 +54,14 @@ const createPreloader = (options = {}) => {
path = new CID(path).toString()
}

if (cache.has(path)) {
// we've preloaded this recently, don't preload it again
return
}

// make sure we don't preload this again any time soon
cache.set(path, true)

const fallbackApiUris = shuffle(apiUris)
let success = false
const now = Date.now()
Expand Down
39 changes: 27 additions & 12 deletions packages/ipfs-core/test/preload.spec.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,19 @@ describe('preload', () => {
after(() => cleanup())
afterEach(() => MockPreloadNode.clearPreloadCids())

it('should not preload content multiple times', async function () {
this.timeout(50 * 1000)
const { cid } = await ipfs.add(uint8ArrayFromString(nanoid()), { preload: false })

await all(ipfs.cat(cid))
await MockPreloadNode.waitForCids(cid)

// should not preload the second time
await MockPreloadNode.clearPreloadCids()
await all(ipfs.cat(cid))
await expect(MockPreloadNode.waitForCids(cid)).to.eventually.be.rejectedWith('Timed out waiting for CIDs to be preloaded')
})

it('should preload content added with add', async function () {
this.timeout(50 * 1000)
const res = await ipfs.add(uint8ArrayFromString(nanoid()))
Expand Down Expand Up @@ -112,7 +125,7 @@ describe('preload', () => {
}, {
path: 'dir0/file2',
content: uint8ArrayFromString(nanoid())
}], { wrapWithDirectory: true }))
}], { wrapWithDirectory: true, preload: false }))

const wrappingDir = res.find(file => file.path === '')
expect(wrappingDir).to.exist()
Expand Down Expand Up @@ -147,12 +160,12 @@ describe('preload', () => {

const [parent, link] = await Promise.all([createNode(), createNode()])

await MockPreloadNode.clearPreloadCids()
const cid = await ipfs.object.patch.addLink(parent.cid, {
Name: 'link',
Hash: link.cid,
Tsize: link.node.size
})

await MockPreloadNode.waitForCids(cid)
})

Expand All @@ -171,27 +184,31 @@ describe('preload', () => {
}]
})

await MockPreloadNode.clearPreloadCids()
const cid = await ipfs.object.patch.rmLink(parentCid, { name: 'link' })
await MockPreloadNode.waitForCids(cid)
})

it('should preload content added with object.patch.setData', async function () {
this.timeout(50 * 1000)
const originalCid = await ipfs.object.put({ Data: uint8ArrayFromString(nanoid()), Links: [] })
await MockPreloadNode.clearPreloadCids()
const patchedCid = await ipfs.object.patch.setData(originalCid, uint8ArrayFromString(nanoid()))
await MockPreloadNode.waitForCids(patchedCid)
})

it('should preload content added with object.patch.appendData', async function () {
this.timeout(50 * 1000)
const originalCid = await ipfs.object.put({ Data: uint8ArrayFromString(nanoid()), Links: [] })
await MockPreloadNode.clearPreloadCids()
const patchedCid = await ipfs.object.patch.appendData(originalCid, uint8ArrayFromString(nanoid()))
await MockPreloadNode.waitForCids(patchedCid)
})

it('should preload content retrieved with object.get', async function () {
this.timeout(50 * 1000)
const cid = await ipfs.object.new({ preload: false })
const cid = await ipfs.object.put({ Data: uint8ArrayFromString(nanoid()), Links: [] }, { preload: false })
await MockPreloadNode.clearPreloadCids()
await ipfs.object.get(cid)
await MockPreloadNode.waitForCids(cid)
})
Expand All @@ -205,13 +222,15 @@ describe('preload', () => {
it('should preload content retrieved with block.get', async function () {
this.timeout(50 * 1000)
const block = await ipfs.block.put(uint8ArrayFromString(nanoid()), { preload: false })
await MockPreloadNode.clearPreloadCids()
await ipfs.block.get(block.cid)
await MockPreloadNode.waitForCids(block.cid)
})

it('should preload content retrieved with block.stat', async function () {
this.timeout(50 * 1000)
const block = await ipfs.block.put(uint8ArrayFromString(nanoid()), { preload: false })
await MockPreloadNode.clearPreloadCids()
await ipfs.block.stat(block.cid)
await MockPreloadNode.waitForCids(block.cid)
})
Expand All @@ -228,39 +247,35 @@ describe('preload', () => {
const obj = { test: nanoid() }
const opts = { format: 'dag-cbor', hashAlg: 'sha2-256', preload: false }
const cid = await ipfs.dag.put(obj, opts)
await MockPreloadNode.clearPreloadCids()
await ipfs.dag.get(cid)
await MockPreloadNode.waitForCids(cid)
})

it('should preload content retrieved with files.ls', async () => {
const res = await ipfs.add({ path: `/t/${nanoid()}`, content: uint8ArrayFromString(nanoid()) })
const res = await ipfs.add({ path: `/t/${nanoid()}`, content: uint8ArrayFromString(nanoid()) }, { preload: false })
const dirCid = res.cid
await MockPreloadNode.waitForCids(dirCid)
await MockPreloadNode.clearPreloadCids()
await all(ipfs.files.ls(`/ipfs/${dirCid}`))
await MockPreloadNode.waitForCids(`/ipfs/${dirCid}`)
})

it('should preload content retrieved with files.ls by CID', async () => {
const res = await ipfs.add({ path: `/t/${nanoid()}`, content: uint8ArrayFromString(nanoid()) })
const res = await ipfs.add({ path: `/t/${nanoid()}`, content: uint8ArrayFromString(nanoid()) }, { preload: false })
const dirCid = res.cid
await MockPreloadNode.waitForCids(dirCid)
await MockPreloadNode.clearPreloadCids()
await all(ipfs.files.ls(dirCid))
await MockPreloadNode.waitForCids(dirCid)
})

it('should preload content retrieved with files.read', async () => {
const { cid } = await ipfs.add(uint8ArrayFromString(nanoid()))
await MockPreloadNode.waitForCids(cid)
const { cid } = await ipfs.add(uint8ArrayFromString(nanoid()), { preload: false })
await MockPreloadNode.clearPreloadCids()
await ipfs.files.read(`/ipfs/${cid}`)
await MockPreloadNode.waitForCids(`/ipfs/${cid}`)
})

it('should preload content retrieved with files.stat', async () => {
const { cid: fileCid } = await ipfs.add(uint8ArrayFromString(nanoid()))
await MockPreloadNode.waitForCids(fileCid)
const { cid: fileCid } = await ipfs.add(uint8ArrayFromString(nanoid()), { preload: false })
await MockPreloadNode.clearPreloadCids()
await ipfs.files.stat(`/ipfs/${fileCid}`)
await MockPreloadNode.waitForCids(`/ipfs/${fileCid}`)
Expand Down

0 comments on commit b5ea76a

Please sign in to comment.