Maintain UTXO Set hash using Muhash index

Author: fjahr

src/init.cpp

@@ -22,6 +22,7 @@
 #include <httpserver.h>
 #include <index/blockfilterindex.h>
 #include <index/txindex.h>
+#include <index/utxosethash.h>
 #include <interfaces/chain.h>
 #include <key.h>
 #include <miner.h>
@@ -166,6 +167,9 @@ void Interrupt(NodeContext& node)
         g_txindex->Interrupt();
     }
     ForEachBlockFilterIndex([](BlockFilterIndex& index) { index.Interrupt(); });
+    if (g_utxo_set_hash) {
+        g_utxo_set_hash->Interrupt();
+    }
 }
 
 void Shutdown(NodeContext& node)
@@ -198,6 +202,7 @@ void Shutdown(NodeContext& node)
     if (node.connman) node.connman->Stop();
     if (g_txindex) g_txindex->Stop();
     ForEachBlockFilterIndex([](BlockFilterIndex& index) { index.Stop(); });
+    if (g_utxo_set_hash) g_utxo_set_hash->Stop();
 
     StopTorControl();
 
@@ -213,6 +218,7 @@ void Shutdown(NodeContext& node)
     node.banman.reset();
     g_txindex.reset();
     DestroyAllBlockFilterIndexes();
+    g_utxo_set_hash.reset();
 
     if (::mempool.IsLoaded() && gArgs.GetArg("-persistmempool", DEFAULT_PERSIST_MEMPOOL)) {
         DumpMempool(::mempool);
@@ -1440,6 +1446,7 @@ bool AppInitMain(NodeContext& node)
         filter_index_cache = max_cache / n_indexes;
         nTotalCache -= filter_index_cache * n_indexes;
     }
+    int64_t utsh_cache = 0;
     int64_t nCoinDBCache = std::min(nTotalCache / 2, (nTotalCache / 4) + (1 << 23)); // use 25%-50% of the remainder for disk cache
     nCoinDBCache = std::min(nCoinDBCache, nMaxCoinsDBCache << 20); // cap total coins db cache
     nTotalCache -= nCoinDBCache;
@@ -1658,6 +1665,9 @@ bool AppInitMain(NodeContext& node)
         GetBlockFilterIndex(filter_type)->Start();
     }
 
+    g_utxo_set_hash = MakeUnique<UtxoSetHash>(utsh_cache, false, fReindex);
+    g_utxo_set_hash->Start();
+
     // ********************************************************* Step 9: load wallet
     for (const auto& client : node.chain_clients) {
         if (!client->load()) {

src/node/coinstats.cpp

@@ -7,32 +7,23 @@
 
 #include <coins.h>
 #include <hash.h>
+#include <index/utxosethash.h>
 #include <serialize.h>
-#include <validation.h>
 #include <uint256.h>
 #include <util/system.h>
+#include <validation.h>
 
 #include <map>
 
 #include <boost/thread.hpp>
 
-
-static void ApplyStats(CCoinsStats &stats, CHashWriter& ss, const uint256& hash, const std::map<uint32_t, Coin>& outputs)
+static void ApplyStats(CCoinsStats &stats, const COutPoint outpoint, const Coin& coin)
 {
-    assert(!outputs.empty());
-    ss << hash;
-    ss << VARINT(outputs.begin()->second.nHeight * 2 + outputs.begin()->second.fCoinBase ? 1u : 0u);
     stats.nTransactions++;
-    for (const auto& output : outputs) {
-        ss << VARINT(output.first + 1);
-        ss << output.second.out.scriptPubKey;
-        ss << VARINT(output.second.out.nValue, VarIntMode::NONNEGATIVE_SIGNED);
-        stats.nTransactionOutputs++;
-        stats.nTotalAmount += output.second.out.nValue;
-        stats.nBogoSize += 32 /* txid */ + 4 /* vout index */ + 4 /* height + coinbase */ + 8 /* amount */ +
-                           2 /* scriptPubKey len */ + output.second.out.scriptPubKey.size() /* scriptPubKey */;
-    }
-    ss << VARINT(0u);
+    stats.nTransactionOutputs++;
+    stats.nTotalAmount += coin.out.nValue;
+    stats.nBogoSize += 32 /* txid */ + 4 /* vout index */ + 4 /* height + coinbase */ + 8 /* amount */ +
+                       2 /* scriptPubKey len */ + coin.out.scriptPubKey.size() /* scriptPubKey */;
 }
 
 //! Calculate statistics about the unspent transaction output set
@@ -41,35 +32,34 @@ bool GetUTXOStats(CCoinsView *view, CCoinsStats &stats)
     std::unique_ptr<CCoinsViewCursor> pcursor(view->Cursor());
     assert(pcursor);
 
-    CHashWriter ss(SER_GETHASH, PROTOCOL_VERSION);
+    uint256 muhash_buf;
+    const CBlockIndex* block_index;
     stats.hashBlock = pcursor->GetBestBlock();
     {
         LOCK(cs_main);
-        stats.nHeight = LookupBlockIndex(stats.hashBlock)->nHeight;
+        block_index = LookupBlockIndex(stats.hashBlock);
     }
-    ss << stats.hashBlock;
-    uint256 prevkey;
-    std::map<uint32_t, Coin> outputs;
+
+    stats.nHeight = block_index->nHeight;
+
+    if (!g_utxo_set_hash->LookupHash(block_index, muhash_buf)) {
+        return false;
+    }
+
+    stats.hashSerialized = muhash_buf;
+
     while (pcursor->Valid()) {
         boost::this_thread::interruption_point();
         COutPoint key;
         Coin coin;
         if (pcursor->GetKey(key) && pcursor->GetValue(coin)) {
-            if (!outputs.empty() && key.hash != prevkey) {
-                ApplyStats(stats, ss, prevkey, outputs);
-                outputs.clear();
-            }
-            prevkey = key.hash;
-            outputs[key.n] = std::move(coin);
+            ApplyStats(stats, key, coin);
         } else {
             return error("%s: unable to read value", __func__);
         }
         pcursor->Next();
     }
-    if (!outputs.empty()) {
-        ApplyStats(stats, ss, prevkey, outputs);
-    }
-    stats.hashSerialized = ss.GetHash();
+
     stats.nDiskSize = view->EstimateSize();
     return true;
 }

src/rpc/blockchain.cpp

@@ -977,7 +977,7 @@ static UniValue gettxoutsetinfo(const JSONRPCRequest& request)
             "  \"transactions\": n,      (numeric) The number of transactions with unspent outputs\n"
             "  \"txouts\": n,            (numeric) The number of unspent transaction outputs\n"
             "  \"bogosize\": n,          (numeric) A meaningless metric for UTXO set size\n"
-            "  \"hash_serialized_2\": \"hash\", (string) The serialized hash\n"
+            "  \"utxo_set_hash\": \"hash\", (string) The serialized hash\n"
             "  \"disk_size\": n,         (numeric) The estimated size of the chainstate on disk\n"
             "  \"total_amount\": x.xxx          (numeric) The total amount\n"
             "}\n"
@@ -1000,7 +1000,7 @@ static UniValue gettxoutsetinfo(const JSONRPCRequest& request)
         ret.pushKV("transactions", (int64_t)stats.nTransactions);
         ret.pushKV("txouts", (int64_t)stats.nTransactionOutputs);
         ret.pushKV("bogosize", (int64_t)stats.nBogoSize);
-        ret.pushKV("hash_serialized_2", stats.hashSerialized.GetHex());
+        ret.pushKV("utxo_set_hash", stats.hashSerialized.GetHex());
         ret.pushKV("disk_size", stats.nDiskSize);
         ret.pushKV("total_amount", ValueFromAmount(stats.nTotalAmount));
     } else {

test/functional/feature_dbcrash.py

@@ -89,7 +89,7 @@ def restart_node(self, node_index, expected_tip):
                 # Any of these RPC calls could throw due to node crash
                 self.start_node(node_index)
                 self.nodes[node_index].waitforblock(expected_tip)
-                utxo_hash = self.nodes[node_index].gettxoutsetinfo()['hash_serialized_2']
+                utxo_hash = self.nodes[node_index].gettxoutsetinfo()['utxo_set_hash']
                 return utxo_hash
             except:
                 # An exception here should mean the node is about to crash.
@@ -134,7 +134,7 @@ def sync_node3blocks(self, block_hashes):
         If any nodes crash while updating, we'll compare utxo hashes to
         ensure recovery was successful."""
 
-        node3_utxo_hash = self.nodes[3].gettxoutsetinfo()['hash_serialized_2']
+        node3_utxo_hash = self.nodes[3].gettxoutsetinfo()['utxo_set_hash']
 
         # Retrieve all the blocks from node3
         blocks = []
@@ -176,12 +176,12 @@ def verify_utxo_hash(self):
         """Verify that the utxo hash of each node matches node3.
 
         Restart any nodes that crash while querying."""
-        node3_utxo_hash = self.nodes[3].gettxoutsetinfo()['hash_serialized_2']
+        node3_utxo_hash = self.nodes[3].gettxoutsetinfo()['utxo_set_hash']
         self.log.info("Verifying utxo hash matches for all nodes")
 
         for i in range(3):
             try:
-                nodei_utxo_hash = self.nodes[i].gettxoutsetinfo()['hash_serialized_2']
+                nodei_utxo_hash = self.nodes[i].gettxoutsetinfo()['utxo_set_hash']
             except OSError:
                 # probably a crash on db flushing
                 nodei_utxo_hash = self.restart_node(i, self.nodes[3].getbestblockhash())

test/functional/rpc_blockchain.py

@@ -216,7 +216,7 @@ def _test_gettxoutsetinfo(self):
         assert size > 6400
         assert size < 64000
         assert_equal(len(res['bestblock']), 64)
-        assert_equal(len(res['hash_serialized_2']), 64)
+        assert_equal(len(res['utxo_set_hash']), 64)
 
         self.log.info("Test that gettxoutsetinfo() works for blockchain with just the genesis block")
         b1hash = node.getblockhash(1)
@@ -229,7 +229,7 @@ def _test_gettxoutsetinfo(self):
         assert_equal(res2['txouts'], 0)
         assert_equal(res2['bogosize'], 0),
         assert_equal(res2['bestblock'], node.getblockhash(0))
-        assert_equal(len(res2['hash_serialized_2']), 64)
+        assert_equal(len(res2['utxo_set_hash']), 64)
 
         self.log.info("Test that gettxoutsetinfo() returns the same result after invalidate/reconsider block")
         node.reconsiderblock(b1hash)