db: add new backend ffldb #331
Conversation
lib/utils/blockio.js
Outdated
|
|
||
| bw.writeU32(this.network.magic); | ||
| bw.writeU32(this.block.length); | ||
| bw.writeBytes(this.block); |
There was a problem hiding this comment.
It would be better to write the block to disk directly and build the CRC incrementally. This prevents a reallocation of 1mb+ just for a write.
lib/utils/blockio.js
Outdated
| BlockIO.prototype.open = function open(file = 0, offset = 0) { | ||
| // TODO: load cursor state from db | ||
| const filepath = this.filepath(file); | ||
| const fd = fs.openSync(filepath, 'a+'); |
There was a problem hiding this comment.
Async calls please: await fs.open(filepath, 'a+')
lib/utils/blockio.js
Outdated
| */ | ||
|
|
||
| BlockIO.prototype.blockFile = async function blockFile(file) { | ||
| if (file in this.files) { |
There was a problem hiding this comment.
No usage of in. Use if (this.files[file] != null) if you need to.
lib/db/ffldb.js
Outdated
| FlatFileDB.prototype.open = function open(options, callback) { | ||
| this.blockio.ensure(); | ||
| this.blockio.open(); | ||
| LevelDOWN.prototype.open.call(this, options, callback); |
There was a problem hiding this comment.
Not good to inherit from Leveldown directly. Use db/lowlevelup instead, to avoid having to call co.promisify repeatedly.
There was a problem hiding this comment.
Updated, no co.promisify anymore.
tuxcanfly
force-pushed
the
treap
branch
2 times, most recently
from
3820798
to
c9fe287
Compare
|
Most recent benchmarks for comparision: There doesn't seem to be any performance improvement in block read times, which was one of the reasons for moving to the flat file backend. @chjj I guess this PR can be closed. |
|
We can explore this more in the future. I feel like maybe node.js's use of |
This PR adds a new db backend,
ffldb, which stores blocks in flat files, using leveldb as a metadata database to store the indexes. The file formatblocks/*.fdbis inspired from btcd and compatibility is maintained.Working on wrapping up some of the remaining cases and scenarios. Will be able to run and share benchmarks soon.
WIP: Not ready for merge