wallet: use height + index for count index

lib/wallet/layout.js

@@ -78,8 +78,8 @@ exports.wdb = {
  *   ---------
  *   g[monotonic-time][hash] -> dummy (tx by monotonic time)
  *   G[account][monotonic-time][hash] -> dummy (tx by monotonic time + account)
- *   z[count] -> dummy (tx by count)
- *   Z[account][count]-> dummy (tx by count + account)
+ *   z[height][index] -> dummy (tx by count)
+ *   Z[account][height][index]-> dummy (tx by count + account)
  *   y[hash] -> count (count for tx)
  *   Y[account][hash] -> count (account count for tx)
  *   h[height][hash] -> dummy (tx by height)
@@ -119,8 +119,8 @@ exports.txdb = {
   // Confirmed
   g: bdb.key('g', ['uint32', 'hash256']),
   G: bdb.key('G', ['uint32', 'uint32', 'hash256']),
-  z: bdb.key('z', ['uint32']),
-  Z: bdb.key('Z', ['uint32', 'uint32']),
+  z: bdb.key('z', ['uint32', 'uint32']),
+  Z: bdb.key('Z', ['uint32', 'uint32', 'uint32']),
   y: bdb.key('y', ['hash256']),
   Y: bdb.key('Y', ['uint32', 'hash256']),
   h: bdb.key('h', ['uint32', 'hash256']),

lib/wallet/records.js

@@ -227,6 +227,63 @@ class BlockMeta {
   }
 }
 
+/**
+ * TX Count
+ */
+
+class TXCount {
+  /**
+   * Create tx count record.
+   * @constructor
+   * @param {TX} tx
+   * @param {BlockMeta?} block
+   */
+
+  constructor(height, index) {
+    this.height = height;
+    this.index = index;
+  }
+
+  /**
+   * Serialize.
+   * @returns {Buffer}
+   */
+
+  toRaw() {
+    const bw = bio.write(8);
+
+    bw.writeU32(this.height);
+    bw.writeU32(this.index);
+
+    return bw.render();
+  }
+
+  /**
+   * Deserialize.
+   * @private
+   * @param {Buffer} data
+   */
+
+  fromRaw(data) {
+    const br = bio.read(data);
+
+    this.height = br.readU32();
+    this.index = br.readU32();
+
+    return this;
+  }
+
+  /**
+   * Instantiate a tx count from a buffer.
+   * @param {Buffer} data
+   * @returns {TXCountRecord}
+   */
+
+  static fromRaw(data) {
+    return new this().fromRaw(data);
+  }
+}
+
 /**
  * TX Record
  */
@@ -497,6 +554,7 @@ class MapRecord {
 
 exports.ChainState = ChainState;
 exports.BlockMeta = BlockMeta;
+exports.TXCount = TXCount;
 exports.TXRecord = TXRecord;
 exports.MapRecord = MapRecord;
 

lib/wallet/rpc.js

@@ -155,7 +155,6 @@ class RPC extends RPCBase {
     this.add('listreceivedbyaddress', this.listReceivedByAddress);
     this.add('listsinceblock', this.listSinceBlock);
     this.add('listtransactions', this.listTransactions);
-    this.add('listhistorycount', this.listHistoryCount);
     this.add('listhistory', this.listHistory);
     this.add('listhistoryafter', this.listHistoryAfter);
     this.add('listhistorybytime', this.listHistoryByTime);
@@ -1223,18 +1222,6 @@ class RPC extends RPCBase {
                     'Use `listhistory` and related methods.');
   }
 
-  async listHistoryCount(args, help) {
-    if (help || args.length > 2) {
-      throw new RPCError(errs.MISC_ERROR,
-        'listhistorycount ( "account" )');
-    }
-    const wallet = this.wallet;
-    const valid = new Validator(args);
-    const name = valid.str(0, 'default');
-
-    return await wallet.getLatestTXCount(name);
-  }
-
   async listHistory(args, help) {
     if (help || args.length > 4) {
       throw new RPCError(errs.MISC_ERROR,

lib/wallet/txdb.js

@@ -19,7 +19,7 @@ const records = require('./records');
 const layout = require('./layout').txdb;
 const consensus = require('../protocol/consensus');
 const policy = require('../protocol/policy');
-const {TXRecord} = records;
+const {TXRecord, TXCount} = records;
 const {inspectSymbol} = require('../utils');
 
 /**
@@ -411,15 +411,20 @@ class TXDB {
    * Add transaction without a batch.
    * @private
    * @param {TX} tx
+   * @param {BlockMeta?} block
+   * @param {Number?} index - The index of txs (not within block)
    * @returns {Promise}
    */
 
-  async add(tx, block) {
+  async add(tx, block, index) {
     const hash = tx.hash();
     const existing = await this.getTX(hash);
 
     assert(!tx.mutable, 'Cannot add mutable TX to wallet.');
 
+    if (block)
+      assert(Number.isInteger(index), 'Index is required with block.');
+
     if (existing) {
       // Existing tx is already confirmed. Ignore.
       if (existing.height !== -1)
@@ -431,7 +436,7 @@ class TXDB {
         return null;
 
       // Confirm transaction.
-      return this.confirm(existing, block);
+      return this.confirm(existing, block, index);
     }
 
     const wtx = TXRecord.fromTX(tx, block);
@@ -447,18 +452,19 @@ class TXDB {
     }
 
     // Finally we can do a regular insertion.
-    return this.insert(wtx, block);
+    return this.insert(wtx, block, index);
   }
 
   /**
    * Insert transaction.
    * @private
    * @param {TXRecord} wtx
-   * @param {BlockMeta} block
+   * @param {BlockMeta?} block
+   * @param {Number?} txindex - The index of txs (not within block)
    * @returns {Promise}
    */
 
-  async insert(wtx, block) {
+  async insert(wtx, block, txindex) {
     const b = this.bucket.batch();
     const {tx, hash} = wtx;
     const height = block ? block.height : -1;
@@ -597,7 +603,13 @@ class TXDB {
 
       // Add count based indexes for transactions that are
       // confirmed however not previously seen.
-      await this.addCountIndex(b, state.accounts, hash);
+      await this.addCountIndex({
+        b,
+        accounts: state.accounts,
+        hash,
+        height: block.height,
+        index: txindex
+      });
 
       // Update block records.
       await this.addBlockMap(b, height);
@@ -633,10 +645,11 @@ class TXDB {
    * @private
    * @param {TXRecord} wtx
    * @param {BlockMeta} block
+   * @param {Number} txindex - The index of txs (not within block)
    * @returns {Promise}
    */
 
-  async confirm(wtx, block) {
+  async confirm(wtx, block, txindex) {
     const b = this.bucket.batch();
     const {tx, hash} = wtx;
     const height = block.height;
@@ -757,7 +770,13 @@ class TXDB {
     // Add count based indexes for transactions
     // that already exist in the database and are now
     // being confirmed.
-    await this.addCountIndex(b, state.accounts, hash);
+    await this.addCountIndex({
+      b,
+      accounts: state.accounts,
+      hash,
+      height: block.height,
+      index: txindex
+    });
 
     // Disconnect unconfirmed count index for the transaction.
     await this.disconnectCountIndexUnconfirmed(b, state.accounts, hash);
@@ -915,20 +934,30 @@ class TXDB {
    * Add count based indexing to support querying
    * transaction history in subsets.
    * @private
-   * @param {Batch} b
-   * @param {Array} accounts
-   * @param {Buffer} hash
+   * @param {Batch} options.b
+   * @param {Array} options.accounts
+   * @param {Buffer} options.hash
+   * @param {Number} options.height
+   * @param {Number} options.index
    */
 
-  async addCountIndex(b, accounts, hash) {
-    const count = await this.getLatestTXCount();
-    b.put(layout.z.encode(count), hash);
-    b.put(layout.y.encode(hash), fromU32BE(count));
+  async addCountIndex(options) {
+    const {
+      b,
+      accounts,
+      hash,
+      height,
+      index
+    } = options;
+
+    const count = new TXCount(height, index);
+
+    b.put(layout.z.encode(height, index), hash);
+    b.put(layout.y.encode(hash), count.toRaw());
 
     for (const [acct,] of accounts) {
-      const acctCount = await this.getLatestTXCount(acct);
-      b.put(layout.Z.encode(acct, acctCount), hash);
-      b.put(layout.Y.encode(acct, hash), fromU32BE(acctCount));
+      b.put(layout.Z.encode(acct, height, index), hash);
+      b.put(layout.Y.encode(acct, hash), count.toRaw());
     }
   }
 
@@ -942,12 +971,12 @@ class TXDB {
 
   async removeCountIndex(b, accounts, hash) {
     const count = await this.getCountForTX(hash);
-    b.del(layout.z.encode(count));
+
+    b.del(layout.z.encode(count.height, count.index));
     b.del(layout.y.encode(hash));
 
     for (const [acct,] of accounts) {
-      const acctCount = await this.getAccountCountForTX(acct, hash);
-      b.del(layout.Z.encode(acct, acctCount));
+      b.del(layout.Z.encode(acct, count.height, count.index));
       b.del(layout.Y.encode(acct, hash));
     }
   }
@@ -1692,43 +1721,6 @@ class TXDB {
     return keys.length > 0 ? keys[0] + 1 : 0;
   }
 
-  /**
-   * Get the latest TX count from the database.
-   * @param {Number?} acct
-   * @returns {Promise} - Returns Number.
-   */
-
-  async getLatestTXCount(acct) {
-    let min, max, parse = null;
-
-    if (!acct) {
-      min = layout.z.min();
-      max = layout.z.max();
-      parse = (key) => {
-        const [index] = layout.z.decode(key);
-        return index;
-      }
-    } else {
-      assert(typeof acct === 'number');
-      min = layout.Z.min(acct);
-      max = layout.Z.max(acct);
-      parse = (key) => {
-        const [,index] = layout.Z.decode(key);
-        return index;
-      }
-    }
-
-    const keys = await this.bucket.keys({
-      gte: min,
-      lte: max,
-      limit: 1,
-      reverse: true, // TODO: Verify that `bdb` supports `reverse`
-      parse: parse
-    });
-
-    return keys.length > 0 ? keys[0] + 1 : 0;
-  }
-
   /**
    * Get TX hashes by height range.
    * @param {Number} acct
@@ -1945,11 +1937,7 @@ class TXDB {
     if (options.limit > 100)
       throw new Error('Limit exceeds max of 100.');
 
-    let count = null;
-    if (acct !== -1)
-      count = await this.getAccountCountForTX(acct, options.txid);
-    else
-      count = await this.getCountForTX(options.txid);
+    const count = await this.getCountForTX(options.txid);
 
     let zopts = {
       limit: options.limit,
@@ -1962,17 +1950,17 @@ class TXDB {
     if (acct !== -1) {
       if (zopts.reverse) {
         zopts['gte'] = layout.Z.min(acct);
-        zopts[lesser] = layout.Z.encode(acct, count);
+        zopts[lesser] = layout.Z.encode(acct, count.height, count.index);
       } else {
-        zopts[greater] = layout.Z.encode(acct, count);
+        zopts[greater] = layout.Z.encode(acct, count.height, count.index);
         zopts['lte'] = layout.Z.max(acct);
       }
     } else {
       if (zopts.reverse) {
         zopts['gte'] = layout.z.min();
-        zopts[lesser] = layout.z.encode(count);
+        zopts[lesser] = layout.z.encode(count.height, count.index);
       } else {
-        zopts[greater] = layout.z.encode(count);
+        zopts[greater] = layout.z.encode(count.height, count.index);
         zopts['lte'] = layout.z.max();
       }
     }
@@ -2187,7 +2175,7 @@ class TXDB {
   /**
    * Get the count of a transaction.
    * @param {Buffer} txid
-   * @returns {Promise} - Returns Number.
+   * @returns {Promise} - Returns TXCount.
    */
 
   async getCountForTX(txid) {
@@ -2196,24 +2184,10 @@ class TXDB {
     const raw = await this.bucket.get(layout.y.encode(txid));
     if (!raw)
       throw new Error('Transaction count not found.');
-    return raw.readUInt32BE(0, true);
-  }
 
-  /**
-   * Get the account count of a transaction.
-   * @param {Number} acct
-   * @param {Buffer} hash
-   * @returns {Promise} - Returns Number.
-   */
-
-  async getAccountCountForTX(acct, hash) {
-    assert(typeof acct === 'number');
-    assert(Buffer.isBuffer(hash));
+    const count = TXCount.fromRaw(raw);
 
-    const raw = await this.bucket.get(layout.Y.encode(acct, hash));
-    if (!raw)
-      throw new Error('Transaction account count not found.');
-    return raw.readUInt32BE(0, true);
+    return count;
   }
 
   /**