db: add new backend ffldb

bench/ffldb.js

@@ -0,0 +1,53 @@
+'use strict';
+
+const assert = require('../test/util/assert');
+const fs = require('../lib/utils/fs');
+const path = require('path');
+const bench = require('./bench');
+const networks = require('../lib/protocol/networks');
+const FlatFileDB = require('../lib/db/ffldb');
+
+const TESTDB = './ffldb-test';
+
+const ffldb = new FlatFileDB(TESTDB, {'network': 'simnet'});
+
+const rm = async (dir) => {
+  const files = await fs.readdir(dir);
+  for (const file of files) {
+    const fp = path.join(dir, file);
+    const stat = await fs.lstat(fp);
+    if (stat.isDirectory()) {
+      await rm(fp);
+    } else {
+      await fs.unlink(fp);
+    }
+  }
+  fs.rmdir(dir);
+};
+
+(async () => {
+  await ffldb.open();
+
+  // Read Block
+  {
+    const key = networks.main.genesis.hash;
+
+    const block = Buffer.from(networks.main.genesisBlock, 'hex');
+    await ffldb.putBlock(key, block);
+
+    const expected = await ffldb.getBlock(key);
+    assert.bufferEqual(expected, block);
+
+    const end = bench('read block');
+
+    for (let i = 0; i < 100000; i++) {
+      await ffldb.getBlock(key);
+    }
+
+    end(100000);
+  }
+
+  await ffldb.close();
+
+  await rm(TESTDB);
+})();

bench/leveldb.js

@@ -0,0 +1,58 @@
+'use strict';
+
+const assert = require('assert');
+const path = require('path');
+const fs = require('../lib/utils/fs');
+const bench = require('./bench');
+const co = require('../lib/utils/co');
+const layout = require('../lib/blockchain/layout');
+const networks = require('../lib/protocol/networks');
+const LevelDOWN = require('leveldown');
+
+const TESTDB = './leveldb-test';
+
+const rm = async (dir) => {
+  const files = await fs.readdir(dir);
+  for (const file of files) {
+    const fp = path.join(dir, file);
+    const stat = await fs.lstat(fp);
+    if (stat.isDirectory()) {
+      rm(fp);
+    } else {
+      await fs.unlink(fp);
+    }
+  }
+  fs.rmdir(dir);
+};
+
+const leveldb = new LevelDOWN(TESTDB);
+
+(async () => {
+  const open = co.promisify(leveldb.open);
+  await open.call(leveldb);
+
+  // Read Block
+  {
+    const key = layout.b(networks.main.genesis.hash);
+    const value = networks.main.genesisBlock;
+
+    const put = co.promisify(leveldb.put);
+    const get = co.promisify(leveldb.get);
+
+    const end = bench('read block');
+    await put.call(leveldb, key, value);
+    const expected = await get.call(leveldb, key);
+    assert.strictEqual(expected.toString(), value);
+
+    for (let i = 0; i < 100000; i++) {
+      await get.call(leveldb, key);
+    }
+
+    end(100000);
+  }
+
+  const close = co.promisify(leveldb.close);
+  await close.call(leveldb);
+
+  await rm(TESTDB);
+})();

lib/blockchain/chaindb.js

@@ -797,7 +797,7 @@ ChainDB.prototype.prune = async function prune() {
     if (!hash)
       throw new Error(`Cannot find hash for ${i}.`);
 
-    batch.del(layout.b(hash));
+    this.db.delBlock(hash);
     batch.del(layout.u(hash));
   }
 
@@ -1073,7 +1073,7 @@ ChainDB.prototype.getRawBlock = async function getRawBlock(block) {
   if (!hash)
     return null;
 
-  return await this.db.get(layout.b(hash));
+  return await this.db.getBlock(hash);
 };
 
 /**
@@ -1093,6 +1093,7 @@ ChainDB.prototype.getBlockView = async function getBlockView(block) {
     const tx = block.txs[i];
 
     for (let j = tx.inputs.length - 1; j >= 0; j--) {
+
       const input = tx.inputs[j];
       undo.apply(view, input.prevout);
     }
@@ -1656,7 +1657,7 @@ ChainDB.prototype._removeChain = async function _removeChain(hash) {
     this.del(layout.p(tip.hash));
     this.del(layout.h(tip.hash));
     this.del(layout.e(tip.hash));
-    this.del(layout.b(tip.hash));
+    this.db.delBlock(hash);
 
     // Queue up hash to be removed
     // on successful write.
@@ -1684,7 +1685,7 @@ ChainDB.prototype.saveBlock = async function saveBlock(entry, block, view) {
 
   // Write actual block data (this may be
   // better suited to flat files in the future).
-  this.put(layout.b(hash), block.toRaw());
+  await this.db.putBlock(hash, block.toRaw());
 
   if (!view)
     return;
@@ -1708,7 +1709,7 @@ ChainDB.prototype.removeBlock = async function removeBlock(entry) {
   if (!block)
     throw new Error('Block not found.');
 
-  this.del(layout.b(block.hash()));
+  this.db.delBlock(block.hash());
 
   return await this.disconnectBlock(entry, block);
 };
@@ -1870,7 +1871,7 @@ ChainDB.prototype.pruneBlock = async function pruneBlock(entry) {
   if (!hash)
     return;
 
-  this.del(layout.b(hash));
+  this.db.delBlock(hash);
   this.del(layout.u(hash));
 };
 

lib/db/backends.js

@@ -17,6 +17,8 @@ exports.get = function get(name) {
         return require('lmdb');
       case 'memory':
         return require('./memdb');
+      case 'ffldb':
+        return require('./ffldb');
       default:
         throw new Error(`Database backend "${name}" not found.`);
     }

lib/db/ffldb.js

@@ -0,0 +1,122 @@
+/*!
+ * ffldb.js - flat file database for bcoin
+ * Copyright (c) 2016-2017, Christopher Jeffrey (MIT License).
+ * https://github.com/bcoin-org/bcoin
+ */
+
+'use strict';
+
+const path = require('path');
+const layout = require('../blockchain/layout');
+const LevelDOWN = require('leveldown');
+const LowLevelUp = require('./lowlevelup');
+const BlockIO = require('../utils/blockio');
+const FileLocation = require('../utils/fileloc');
+
+const METADATA = 'metadata.ldb';
+
+/**
+ * Flat file database for bcoin
+ * using a leveldb backend for metadata.
+ * @alias module:db.FlatFileDB
+ * @constructor
+ * @param {String} location
+ * @param {Object?} options
+ */
+
+function FlatFileDB(location, options) {
+  if (!(this instanceof FlatFileDB))
+    return new FlatFileDB(location, options);
+
+  LowLevelUp.call(this, LevelDOWN, path.join(location, METADATA), options);
+
+  this.location = location;
+  // TODO: Keep coin, tx in memory using @Treap
+
+  this.blockio = new BlockIO({
+    location: path.join(location, 'blocks'),
+    maxFileSize: 512 * 1024 * 1024, // 512 MiB
+    network: options.network
+  });
+}
+
+Object.setPrototypeOf(FlatFileDB.prototype, LowLevelUp.prototype);
+
+/**
+ * Retrieve a raw block by hash
+ * @param {Hash} hash
+ * @returns {Promise} - Returns Buffer.
+ */
+
+FlatFileDB.prototype.getBlock = async function getBlock(key) {
+  try {
+    const entry = await this.get(layout.b(key));
+    const loc = FileLocation.fromRaw(entry);
+    return this.blockio.readBlock(loc);
+  } catch (e) {
+    throw e;
+  }
+};
+
+/**
+ * Store a raw block
+ * @param {Hash} key
+ * @param {Buffer} raw block
+ */
+
+FlatFileDB.prototype.putBlock = async function putBlock(key, value) {
+  try {
+    const loc = await this.blockio.writeBlock(value);
+    const entry = loc.toRaw();
+    this.put(layout.b(key), entry);
+  } catch (e) {
+    throw e;
+  }
+};
+
+/**
+ * Remove a block by hash
+ * @param {Hash} key
+ */
+
+FlatFileDB.prototype.delBlock = async function delBlock(key) {
+  try {
+    const entry = await this.get(layout.b(key));
+    if (!entry) {
+      return;
+    }
+    const loc = FileLocation.fromRaw(entry);
+    await this.blockio.removeBlock(loc);
+    // TODO: delete block keys from metadata db
+  } catch (e) {
+    throw e;
+  }
+};
+
+/**
+ * Open the database (leveldown method).
+ * @param {Object} options
+ * @param {Function} callback
+ */
+
+FlatFileDB.prototype.open = async function open() {
+  await this.blockio.ensure();
+  await this.blockio.open();
+  await LowLevelUp.prototype.open.call(this);
+};
+
+/**
+ * Close the database (leveldown method).
+ * @param {Function} callback
+ */
+
+FlatFileDB.prototype.close = async function close() {
+  await this.blockio.close();
+  await LowLevelUp.prototype.close.call(this);
+};
+
+/*
+ * Expose
+ */
+
+module.exports = FlatFileDB;

lib/db/index.js

@@ -14,3 +14,4 @@ exports.backends = require('./backends');
 exports.LDB = require('./ldb');
 exports.LowlevelUp = require('./lowlevelup');
 exports.MemDB = require('./memdb');
+exports.FlatFile = require('./ffldb');

lib/db/ldb.js

@@ -23,6 +23,11 @@ function LDB(options) {
   const backend = result.backend;
   const location = result.location;
 
+  if (backend.prototype instanceof LowlevelUp) {
+    const Backend = backend;
+    return new Backend(location, options);
+  }
+
   return new LowlevelUp(backend, location, options);
 }
 
@@ -62,6 +67,10 @@ LDB.getName = function getName(db) {
       name = 'memory';
       ext = 'mem';
       break;
+    case 'ffldb':
+      name = 'ffldb';
+      ext = 'fdb';
+      break;
     default:
       name = db;
       ext = 'db';