Skip to content
This repository was archived by the owner on Aug 11, 2021. It is now read-only.

feat: dynamic format loading #178

Merged
merged 2 commits into from
Nov 9, 2018
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
20 changes: 20 additions & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -130,6 +130,26 @@ const ipld = new Ipld({
})
```

##### `options.loadFormat(codec, callback)`

| Type | Default |
|------|---------|
| `Function` | `null` |

Function to dynamically load an [IPLD Format](https://github.com/ipld/interface-ipld-format). It is passed a string `codec`, the multicodec of the IPLD format to load and a callback function to call when the format has been loaded. e.g.

```js
const ipld = new Ipld({
loadFormat (codec, callback) {
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Wow, I didn't know that this syntax is a things. It's nice to learn new things by code reviewing.

if (codec === 'git-raw') {
callback(null, require('ipld-git'))
} else {
callback(new Error('unable to load format ' + codec))
}
}
})
```

### `.put(node, options, callback)`

> Store the given node of a recognized IPLD Format.
Expand Down
152 changes: 74 additions & 78 deletions src/index.js
Original file line number Diff line number Diff line change
Expand Up @@ -48,6 +48,10 @@ class IPLDResolver {
}
}

this.support.load = options.loadFormat || ((codec, callback) => {
callback(new Error(`No resolver found for codec "${codec}"`))
})

this.support.rm = (multicodec) => {
if (this.resolvers[multicodec]) {
delete this.resolvers[multicodec]
Expand Down Expand Up @@ -99,26 +103,24 @@ class IPLDResolver {

doUntil(
(cb) => {
const r = this.resolvers[cid.codec]

if (!r) {
return cb(new Error('No resolver found for codec "' + cid.codec + '"'))
}
this._getFormat(cid.codec, (err, format) => {
if (err) return cb(err)

// get block
// use local resolver
// update path value
this.bs.get(cid, (err, block) => {
if (err) {
return cb(err)
}
r.resolver.resolve(block.data, path, (err, result) => {
// get block
// use local resolver
// update path value
this.bs.get(cid, (err, block) => {
if (err) {
return cb(err)
}
value = result.value
path = result.remainderPath
cb()
format.resolver.resolve(block.data, path, (err, result) => {
if (err) {
return cb(err)
}
value = result.value
path = result.remainderPath
cb()
})
})
})
},
Expand Down Expand Up @@ -182,17 +184,9 @@ class IPLDResolver {
return callback(err)
}
map(blocks, (block, mapCallback) => {
const resolver = this.resolvers[block.cid.codec]
if (!resolver) {
return mapCallback(
new Error('No resolver found for codec "' + block.cid.codec + '"'))
}

resolver.util.deserialize(block.data, (err, deserialized) => {
if (err) {
return mapCallback(err)
}
return mapCallback(null, deserialized)
this._getFormat(block.cid.codec, (err, format) => {
if (err) return mapCallback(err)
format.util.deserialize(block.data, mapCallback)
})
},
callback)
Expand All @@ -216,20 +210,20 @@ class IPLDResolver {
return this._put(options.cid, node, callback)
}

const r = this.resolvers[options.format]
if (!r) {
return callback(new Error('No resolver found for codec "' + options.format + '"'))
}
r.util.cid(node, options, (err, cid) => {
if (err) {
return callback(err)
}
this._getFormat(options.format, (err, format) => {
if (err) return callback(err)

if (options.onlyHash) {
return callback(null, cid)
}
format.util.cid(node, options, (err, cid) => {
if (err) {
return callback(err)
}

this._put(cid, node, callback)
if (options.onlyHash) {
return callback(null, cid)
}

this._put(cid, node, callback)
})
})
}

Expand All @@ -245,15 +239,14 @@ class IPLDResolver {

if (!options.recursive) {
p = pullDeferSource()
const r = this.resolvers[cid.codec]
if (!r) {
p.abort(new Error('No resolver found for codec "' + cid.codec + '"'))
return p
}

waterfall([
(cb) => this.bs.get(cid, cb),
(block, cb) => r.resolver.tree(block.data, cb)
(cb) => this._getFormat(cid.codec, cb),
(format, cb) => this.bs.get(cid, (err, block) => {
if (err) return cb(err)
cb(null, format, block)
}),
(format, block, cb) => format.resolver.tree(block.data, cb)
], (err, paths) => {
if (err) {
p.abort(err)
Expand All @@ -280,20 +273,19 @@ class IPLDResolver {

const deferred = pullDeferSource()
const cid = el.cid
const r = this.resolvers[cid.codec]
if (!r) {
deferred.abort(new Error('No resolver found for codec "' + cid.codec + '"'))
return deferred
}

waterfall([
(cb) => this.bs.get(el.cid, cb),
(block, cb) => r.resolver.tree(block.data, (err, paths) => {
(cb) => this._getFormat(cid.codec, cb),
(format, cb) => this.bs.get(cid, (err, block) => {
if (err) return cb(err)
cb(null, format, block)
}),
(format, block, cb) => format.resolver.tree(block.data, (err, paths) => {
if (err) {
return cb(err)
}
map(paths, (p, cb) => {
r.resolver.isLink(block.data, p, (err, link) => {
format.resolver.isLink(block.data, p, (err, link) => {
if (err) {
return cb(err)
}
Expand Down Expand Up @@ -356,38 +348,42 @@ class IPLDResolver {
/* */

_get (cid, callback) {
const r = this.resolvers[cid.codec]
if (!r) {
return callback(new Error('No resolver found for codec "' + cid.codec + '"'))
}

waterfall([
(cb) => this.bs.get(cid, cb),
(block, cb) => {
if (r) {
r.util.deserialize(block.data, (err, deserialized) => {
if (err) {
return cb(err)
}
cb(null, deserialized)
})
} else { // multicodec unknown, send back raw data
cb(null, block.data)
}
(cb) => this._getFormat(cid.codec, cb),
(format, cb) => this.bs.get(cid, (err, block) => {
if (err) return cb(err)
cb(null, format, block)
}),
(format, block, cb) => {
format.util.deserialize(block.data, (err, deserialized) => {
if (err) {
return cb(err)
}
cb(null, deserialized)
})
}
], callback)
}

_getFormat (codec, callback) {
if (this.resolvers[codec]) {
return callback(null, this.resolvers[codec])
}

// If not supported, attempt to dynamically load this format
this.support.load(codec, (err, format) => {
if (err) return callback(err)
this.resolvers[codec] = format
callback(null, format)
})
}

_put (cid, node, callback) {
callback = callback || noop

const r = this.resolvers[cid.codec]
if (!r) {
return callback(new Error('No resolver found for codec "' + cid.codec + '"'))
}

waterfall([
(cb) => r.util.serialize(node, cb),
(cb) => this._getFormat(cid.codec, cb),
(format, cb) => format.util.serialize(node, cb),
(buf, cb) => this.bs.put(new Block(buf, cid), cb)
], (err) => {
if (err) {
Expand Down Expand Up @@ -425,7 +421,7 @@ IPLDResolver.defaultOptions = {
}

/**
* Create an IPLD resolver with an inmemory blockservice and
* Create an IPLD resolver with an in memory blockservice and
* repo.
*
* @param {function(Error, IPLDResolver)} callback
Expand Down
1 change: 1 addition & 0 deletions test/browser.js
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,7 @@ describe('Browser', () => {
})

require('./basics')(repo)
require('./format-support')(repo)
require('./ipld-dag-pb')(repo)
require('./ipld-dag-cbor')(repo)
require('./ipld-git')(repo)
Expand Down
100 changes: 100 additions & 0 deletions test/format-support.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,100 @@
/* eslint-env mocha */
'use strict'

const chai = require('chai')
const dirtyChai = require('dirty-chai')
const expect = chai.expect
chai.use(dirtyChai)
const BlockService = require('ipfs-block-service')
const dagCBOR = require('ipld-dag-cbor')

const IPLDResolver = require('../src')

module.exports = (repo) => {
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Would it make sense to add a test that a format is not dynamically loaded when it was already statically added?

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Done!

describe('IPLD format support', () => {
let data, cid

before((done) => {
const bs = new BlockService(repo)
const resolver = new IPLDResolver({ blockService: bs })

data = { now: Date.now() }
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Why the Date.now()? I generally prefer tests that do the exact same across runs.


dagCBOR.util.cid(data, (err, c) => {
expect(err).to.not.exist()
cid = c
resolver.put(data, { cid }, done)
})
})

describe('Dynamic format loading', () => {
it('should fail to dynamically load format', (done) => {
const bs = new BlockService(repo)
const resolver = new IPLDResolver({
blockService: bs,
formats: []
})

resolver.get(cid, '/', (err) => {
expect(err).to.exist()
expect(err.message).to.equal('No resolver found for codec "dag-cbor"')
done()
})
})

it('should fail to dynamically load format via loadFormat option', (done) => {
const errMsg = 'BOOM' + Date.now()
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Why the Date.now()? I generally prefer tests that do the exact same across runs.

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

In this case I want to ensure the error that was thrown was the error that I threw. I'm asserting on the error message so I need a value that is not going to cause a false positive with any other error that could be thrown.

So, yes, I guess I could just use a static string but there's a tiny bit more risk of it causing a false possitive in the future than a value that is randomly generated.

In general I know that tests are not isolated from side effects, so I like to know my tests are dealing with their own unique data. As an example we share block stores across multiple tests and they don't clean up after themselves. By the way, I don't believe they should clean up and we can't guarantee that it'll happen (in the case of failures) or that unintentional side effects don't happen. Ideally we'd use a new block store for every test, but I appreciate that's not always practical....The point is, using randomly generated data is one way of better isolating tests from side effects caused by other tests.

A completely contrived example: I'm testing I can put a value with IPLD and I use a static value. Many weeks later someone adds a test before mine that puts that same value in the block store. Since the block store is shared, when my test runs the code somehow determines the value is already in the store and returns success. Same result, but it's a different code path that's being tested now, and maybe that's masking a error with the code path that writes to disk...

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

BTW, I'm happy to change if you disagree or would prefer!

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Thanks for the explanation. It makes sense, happy to keep it that way.

const bs = new BlockService(repo)
const resolver = new IPLDResolver({
blockService: bs,
formats: [],
loadFormat (codec, callback) {
if (codec !== 'dag-cbor') return callback(new Error('unexpected codec'))
setTimeout(() => callback(new Error(errMsg)))
}
})

resolver.get(cid, '/', (err) => {
expect(err).to.exist()
expect(err.message).to.equal(errMsg)
done()
})
})

it('should dynamically load missing format', (done) => {
const bs = new BlockService(repo)
const resolver = new IPLDResolver({
blockService: bs,
formats: [],
loadFormat (codec, callback) {
if (codec !== 'dag-cbor') return callback(new Error('unexpected codec'))
setTimeout(() => callback(null, dagCBOR))
}
})

resolver.get(cid, '/', (err, result) => {
expect(err).to.not.exist()
expect(result.value).to.eql(data)
done()
})
})

it('should not dynamically load format added statically', (done) => {
const bs = new BlockService(repo)
const resolver = new IPLDResolver({
blockService: bs,
formats: [dagCBOR],
loadFormat (codec) {
throw new Error(`unexpected load format ${codec}`)
}
})

resolver.get(cid, '/', (err, result) => {
expect(err).to.not.exist()
expect(result.value).to.eql(data)
done()
})
})
})
})
}
1 change: 1 addition & 0 deletions test/node.js
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@ describe('Node.js', () => {
})

require('./basics')(repo)
require('./format-support')(repo)
require('./ipld-dag-pb')(repo)
require('./ipld-dag-cbor')(repo)
require('./ipld-git')(repo)
Expand Down