diff --git a/lib/wallet/layout.js b/lib/wallet/layout.js index dc89e8c7a..55ee10319 100644 --- a/lib/wallet/layout.js +++ b/lib/wallet/layout.js @@ -78,8 +78,8 @@ exports.wdb = { * --------- * g[monotonic-time][hash] -> dummy (tx by monotonic time) * G[account][monotonic-time][hash] -> dummy (tx by monotonic time + account) - * z[count] -> dummy (tx by count) - * Z[account][count]-> dummy (tx by count + account) + * z[height][index] -> dummy (tx by count) + * Z[account][height][index]-> dummy (tx by count + account) * y[hash] -> count (count for tx) * Y[account][hash] -> count (account count for tx) * h[height][hash] -> dummy (tx by height) @@ -119,8 +119,8 @@ exports.txdb = { // Confirmed g: bdb.key('g', ['uint32', 'hash256']), G: bdb.key('G', ['uint32', 'uint32', 'hash256']), - z: bdb.key('z', ['uint32']), - Z: bdb.key('Z', ['uint32', 'uint32']), + z: bdb.key('z', ['uint32', 'uint32']), + Z: bdb.key('Z', ['uint32', 'uint32', 'uint32']), y: bdb.key('y', ['hash256']), Y: bdb.key('Y', ['uint32', 'hash256']), h: bdb.key('h', ['uint32', 'hash256']), diff --git a/lib/wallet/records.js b/lib/wallet/records.js index 778e7b397..f519d9916 100644 --- a/lib/wallet/records.js +++ b/lib/wallet/records.js @@ -227,6 +227,63 @@ class BlockMeta { } } +/** + * TX Count + */ + +class TXCount { + /** + * Create tx count record. + * @constructor + * @param {TX} tx + * @param {BlockMeta?} block + */ + + constructor(height, index) { + this.height = height; + this.index = index; + } + + /** + * Serialize. + * @returns {Buffer} + */ + + toRaw() { + const bw = bio.write(8); + + bw.writeU32(this.height); + bw.writeU32(this.index); + + return bw.render(); + } + + /** + * Deserialize. + * @private + * @param {Buffer} data + */ + + fromRaw(data) { + const br = bio.read(data); + + this.height = br.readU32(); + this.index = br.readU32(); + + return this; + } + + /** + * Instantiate a tx count from a buffer. + * @param {Buffer} data + * @returns {TXCountRecord} + */ + + static fromRaw(data) { + return new this().fromRaw(data); + } +} + /** * TX Record */ @@ -497,6 +554,7 @@ class MapRecord { exports.ChainState = ChainState; exports.BlockMeta = BlockMeta; +exports.TXCount = TXCount; exports.TXRecord = TXRecord; exports.MapRecord = MapRecord; diff --git a/lib/wallet/rpc.js b/lib/wallet/rpc.js index cc5aad0ff..6b65a5bca 100644 --- a/lib/wallet/rpc.js +++ b/lib/wallet/rpc.js @@ -155,7 +155,6 @@ class RPC extends RPCBase { this.add('listreceivedbyaddress', this.listReceivedByAddress); this.add('listsinceblock', this.listSinceBlock); this.add('listtransactions', this.listTransactions); - this.add('listhistorycount', this.listHistoryCount); this.add('listhistory', this.listHistory); this.add('listhistoryafter', this.listHistoryAfter); this.add('listhistorybytime', this.listHistoryByTime); @@ -1223,18 +1222,6 @@ class RPC extends RPCBase { 'Use `listhistory` and related methods.'); } - async listHistoryCount(args, help) { - if (help || args.length > 2) { - throw new RPCError(errs.MISC_ERROR, - 'listhistorycount ( "account" )'); - } - const wallet = this.wallet; - const valid = new Validator(args); - const name = valid.str(0, 'default'); - - return await wallet.getLatestTXCount(name); - } - async listHistory(args, help) { if (help || args.length > 4) { throw new RPCError(errs.MISC_ERROR, diff --git a/lib/wallet/txdb.js b/lib/wallet/txdb.js index d73b3a6e8..15cf6b59f 100644 --- a/lib/wallet/txdb.js +++ b/lib/wallet/txdb.js @@ -19,7 +19,7 @@ const records = require('./records'); const layout = require('./layout').txdb; const consensus = require('../protocol/consensus'); const policy = require('../protocol/policy'); -const {TXRecord} = records; +const {TXRecord, TXCount} = records; const {inspectSymbol} = require('../utils'); /** @@ -411,15 +411,20 @@ class TXDB { * Add transaction without a batch. * @private * @param {TX} tx + * @param {BlockMeta?} block + * @param {Number?} index - The index of txs (not within block) * @returns {Promise} */ - async add(tx, block) { + async add(tx, block, index) { const hash = tx.hash(); const existing = await this.getTX(hash); assert(!tx.mutable, 'Cannot add mutable TX to wallet.'); + if (block) + assert(Number.isInteger(index), 'Index is required with block.'); + if (existing) { // Existing tx is already confirmed. Ignore. if (existing.height !== -1) @@ -431,7 +436,7 @@ class TXDB { return null; // Confirm transaction. - return this.confirm(existing, block); + return this.confirm(existing, block, index); } const wtx = TXRecord.fromTX(tx, block); @@ -447,18 +452,19 @@ class TXDB { } // Finally we can do a regular insertion. - return this.insert(wtx, block); + return this.insert(wtx, block, index); } /** * Insert transaction. * @private * @param {TXRecord} wtx - * @param {BlockMeta} block + * @param {BlockMeta?} block + * @param {Number?} txindex - The index of txs (not within block) * @returns {Promise} */ - async insert(wtx, block) { + async insert(wtx, block, txindex) { const b = this.bucket.batch(); const {tx, hash} = wtx; const height = block ? block.height : -1; @@ -597,7 +603,13 @@ class TXDB { // Add count based indexes for transactions that are // confirmed however not previously seen. - await this.addCountIndex(b, state.accounts, hash); + await this.addCountIndex({ + b, + accounts: state.accounts, + hash, + height: block.height, + index: txindex + }); // Update block records. await this.addBlockMap(b, height); @@ -633,10 +645,11 @@ class TXDB { * @private * @param {TXRecord} wtx * @param {BlockMeta} block + * @param {Number} txindex - The index of txs (not within block) * @returns {Promise} */ - async confirm(wtx, block) { + async confirm(wtx, block, txindex) { const b = this.bucket.batch(); const {tx, hash} = wtx; const height = block.height; @@ -757,7 +770,13 @@ class TXDB { // Add count based indexes for transactions // that already exist in the database and are now // being confirmed. - await this.addCountIndex(b, state.accounts, hash); + await this.addCountIndex({ + b, + accounts: state.accounts, + hash, + height: block.height, + index: txindex + }); // Disconnect unconfirmed count index for the transaction. await this.disconnectCountIndexUnconfirmed(b, state.accounts, hash); @@ -915,20 +934,30 @@ class TXDB { * Add count based indexing to support querying * transaction history in subsets. * @private - * @param {Batch} b - * @param {Array} accounts - * @param {Buffer} hash + * @param {Batch} options.b + * @param {Array} options.accounts + * @param {Buffer} options.hash + * @param {Number} options.height + * @param {Number} options.index */ - async addCountIndex(b, accounts, hash) { - const count = await this.getLatestTXCount(); - b.put(layout.z.encode(count), hash); - b.put(layout.y.encode(hash), fromU32BE(count)); + async addCountIndex(options) { + const { + b, + accounts, + hash, + height, + index + } = options; + + const count = new TXCount(height, index); + + b.put(layout.z.encode(height, index), hash); + b.put(layout.y.encode(hash), count.toRaw()); for (const [acct,] of accounts) { - const acctCount = await this.getLatestTXCount(acct); - b.put(layout.Z.encode(acct, acctCount), hash); - b.put(layout.Y.encode(acct, hash), fromU32BE(acctCount)); + b.put(layout.Z.encode(acct, height, index), hash); + b.put(layout.Y.encode(acct, hash), count.toRaw()); } } @@ -942,12 +971,12 @@ class TXDB { async removeCountIndex(b, accounts, hash) { const count = await this.getCountForTX(hash); - b.del(layout.z.encode(count)); + + b.del(layout.z.encode(count.height, count.index)); b.del(layout.y.encode(hash)); for (const [acct,] of accounts) { - const acctCount = await this.getAccountCountForTX(acct, hash); - b.del(layout.Z.encode(acct, acctCount)); + b.del(layout.Z.encode(acct, count.height, count.index)); b.del(layout.Y.encode(acct, hash)); } } @@ -1692,43 +1721,6 @@ class TXDB { return keys.length > 0 ? keys[0] + 1 : 0; } - /** - * Get the latest TX count from the database. - * @param {Number?} acct - * @returns {Promise} - Returns Number. - */ - - async getLatestTXCount(acct) { - let min, max, parse = null; - - if (!acct) { - min = layout.z.min(); - max = layout.z.max(); - parse = (key) => { - const [index] = layout.z.decode(key); - return index; - } - } else { - assert(typeof acct === 'number'); - min = layout.Z.min(acct); - max = layout.Z.max(acct); - parse = (key) => { - const [,index] = layout.Z.decode(key); - return index; - } - } - - const keys = await this.bucket.keys({ - gte: min, - lte: max, - limit: 1, - reverse: true, // TODO: Verify that `bdb` supports `reverse` - parse: parse - }); - - return keys.length > 0 ? keys[0] + 1 : 0; - } - /** * Get TX hashes by height range. * @param {Number} acct @@ -1945,11 +1937,7 @@ class TXDB { if (options.limit > 100) throw new Error('Limit exceeds max of 100.'); - let count = null; - if (acct !== -1) - count = await this.getAccountCountForTX(acct, options.txid); - else - count = await this.getCountForTX(options.txid); + const count = await this.getCountForTX(options.txid); let zopts = { limit: options.limit, @@ -1962,17 +1950,17 @@ class TXDB { if (acct !== -1) { if (zopts.reverse) { zopts['gte'] = layout.Z.min(acct); - zopts[lesser] = layout.Z.encode(acct, count); + zopts[lesser] = layout.Z.encode(acct, count.height, count.index); } else { - zopts[greater] = layout.Z.encode(acct, count); + zopts[greater] = layout.Z.encode(acct, count.height, count.index); zopts['lte'] = layout.Z.max(acct); } } else { if (zopts.reverse) { zopts['gte'] = layout.z.min(); - zopts[lesser] = layout.z.encode(count); + zopts[lesser] = layout.z.encode(count.height, count.index); } else { - zopts[greater] = layout.z.encode(count); + zopts[greater] = layout.z.encode(count.height, count.index); zopts['lte'] = layout.z.max(); } } @@ -2187,7 +2175,7 @@ class TXDB { /** * Get the count of a transaction. * @param {Buffer} txid - * @returns {Promise} - Returns Number. + * @returns {Promise} - Returns TXCount. */ async getCountForTX(txid) { @@ -2196,24 +2184,10 @@ class TXDB { const raw = await this.bucket.get(layout.y.encode(txid)); if (!raw) throw new Error('Transaction count not found.'); - return raw.readUInt32BE(0, true); - } - /** - * Get the account count of a transaction. - * @param {Number} acct - * @param {Buffer} hash - * @returns {Promise} - Returns Number. - */ - - async getAccountCountForTX(acct, hash) { - assert(typeof acct === 'number'); - assert(Buffer.isBuffer(hash)); + const count = TXCount.fromRaw(raw); - const raw = await this.bucket.get(layout.Y.encode(acct, hash)); - if (!raw) - throw new Error('Transaction account count not found.'); - return raw.readUInt32BE(0, true); + return count; } /** diff --git a/lib/wallet/wallet.js b/lib/wallet/wallet.js index a91bc1c4e..ee9d6115a 100644 --- a/lib/wallet/wallet.js +++ b/lib/wallet/wallet.js @@ -1839,13 +1839,15 @@ class Wallet extends EventEmitter { /** * Add a transaction to the wallets TX history. * @param {TX} tx + * @param {BlockMeta?} block + * @param {Number?} index - The index of txs (not within block) * @returns {Promise} */ - async add(tx, block) { + async add(tx, block, index) { const unlock = await this.writeLock.lock(); try { - return await this._add(tx, block); + return await this._add(tx, block, index); } finally { unlock(); } @@ -1856,11 +1858,13 @@ class Wallet extends EventEmitter { * Potentially resolves orphans. * @private * @param {TX} tx + * @param {BlockMeta?} block + * @param {Number?} index - The index of txs (not within block) * @returns {Promise} */ - async _add(tx, block) { - const details = await this.txdb.add(tx, block); + async _add(tx, block, index) { + const details = await this.txdb.add(tx, block, index); if (details) { const derived = await this.syncOutputDepth(tx); diff --git a/lib/wallet/walletdb.js b/lib/wallet/walletdb.js index 74d8ec858..b466a5bee 100644 --- a/lib/wallet/walletdb.js +++ b/lib/wallet/walletdb.js @@ -2020,7 +2020,7 @@ class WalletDB extends EventEmitter { let total = 0; for (const tx of txs) { - if (await this._addTX(tx, tip)) + if (await this._addTX(tx, tip, total)) total += 1; } @@ -2237,7 +2237,7 @@ class WalletDB extends EventEmitter { * @returns {Promise} */ - async _addTX(tx, block) { + async _addTX(tx, block, index) { const wids = await this.getWalletsByTX(tx); assert(!tx.mutable, 'WDB: Cannot add mutable TX.'); @@ -2261,7 +2261,7 @@ class WalletDB extends EventEmitter { assert(wallet); - if (await wallet.add(tx, block)) { + if (await wallet.add(tx, block, index)) { this.logger.info( 'Added transaction to wallet in WalletDB: %s (%d).', wallet.id, wid); diff --git a/test/wallet-pagination-test.js b/test/wallet-pagination-test.js index a2244306a..c4f718a97 100644 --- a/test/wallet-pagination-test.js +++ b/test/wallet-pagination-test.js @@ -80,11 +80,6 @@ describe('Wallet TX Pagination', function() { await spvnode.close(); }); - it('should get correct transaction count', async () => { - const count = await wclient.execute('listhistorycount', ['blue']); - assert.strictEqual(count, 575); - }); - describe('get transaction history (dsc)', function() { it('first page', async () => { const history = await wclient.execute('listhistory', ['blue', 100, true]); @@ -114,7 +109,7 @@ describe('Wallet TX Pagination', function() { // TODO // - third page after new block (no shifting) - // - last page + // - last page (575 txs total) }); describe('get transaction history (asc)', () => { diff --git a/test/wallet-rescan-test.js b/test/wallet-rescan-test.js index da23c64ff..bab61c064 100644 --- a/test/wallet-rescan-test.js +++ b/test/wallet-rescan-test.js @@ -85,9 +85,6 @@ describe('Wallet Rescan', function() { // TODO remove this await sleep(5000); - const count = await wclient.execute('listhistorycount', ['blue']); - assert.strictEqual(count, 0); - key1 = KeyRing.generate(); key2 = KeyRing.generate(); key3 = KeyRing.generate(); @@ -162,8 +159,8 @@ describe('Wallet Rescan', function() { describe('full node wallet', function() { it('has the correct number of txs', async () => { - const count = await wclient.execute('listhistorycount', ['blue']); - assert.strictEqual(count, 14); + const history = await wclient.execute('listhistory', ['blue', 100, true]); + assert.strictEqual(history.length, 14); }); it('wallet should include txs of imported addresses', async () => { @@ -172,8 +169,8 @@ describe('Wallet Rescan', function() { describe('spv node wallet', function() { it('has the correct number of txs', async () => { - const count = await spvwclient.execute('listhistorycount', ['blue']); - assert.strictEqual(count, 14); + const history = await spvwclient.execute('listhistory', ['blue', 100, true]); + assert.strictEqual(history.length, 14); }); it('wallet should include txs of imported addresses', async () => { diff --git a/test/wallet-unconfirmed-test.js b/test/wallet-unconfirmed-test.js index a2bd87a3f..e4d72a036 100644 --- a/test/wallet-unconfirmed-test.js +++ b/test/wallet-unconfirmed-test.js @@ -110,11 +110,6 @@ describe('Wallet Unconfirmed TX', function() { await spvnode.close(); }); - it('should get correct transaction count', async () => { - const count = await wclient.execute('listhistorycount', ['blue']); - assert.strictEqual(count, 575); - }); - describe('get unconfirmed transaction history (dsc)', function() { it('first page', async () => { const history = await wclient.execute('listunconfirmed', ['blue', 100, true]);