Experimental: Block Filter Digest support.

qa/rpc-tests/test_framework/mininode.py

@@ -50,6 +50,8 @@
 NODE_GETUTXO = (1 << 1)
 NODE_BLOOM = (1 << 2)
 NODE_WITNESS = (1 << 3)
+NODE_XTHIN = (1 << 4)
+NODE_BFD = (1 << 5)
 
 # Keep our own socket map for asyncore, so that we can track disconnects
 # ourselves (to workaround an issue with closing an asyncore socket when

src/Makefile.am

@@ -80,6 +80,7 @@ BITCOIN_CORE_H = \
   addrman.h \
   base58.h \
   bloom.h \
+  blockdigest.h \
   blockencodings.h \
   chain.h \
   chainparams.h \
@@ -178,6 +179,7 @@ libbitcoin_server_a_SOURCES = \
   addrman.cpp \
   addrdb.cpp \
   bloom.cpp \
+  blockdigest.cpp \
   blockencodings.cpp \
   chain.cpp \
   checkpoints.cpp \

src/blockdigest.cpp

@@ -0,0 +1,87 @@
+// Copyright (c) 2017 The Bitcoin Core developers
+// Distributed under the MIT software license, see the accompanying
+// file COPYING or http://www.opensource.org/licenses/mit-license.php.
+
+#include "blockdigest.h"
+#include "streams.h"
+#include "version.h"
+#include "uint256.h"
+#include "primitives/transaction.h"
+#include "utiltime.h"
+#include "utilstrencodings.h"
+
+namespace DigestFilter {
+
+std::vector<uint8_t> base_filter::outpoint_data(const COutPoint& outpoint) const
+{
+    CDataStream stream(SER_NETWORK, PROTOCOL_VERSION);
+    stream << outpoint;
+    return std::vector<uint8_t>(stream.begin(), stream.end());
+}
+
+std::vector<std::vector<uint8_t>> base_filter::tx_data_vec(const CTransaction& tx) const
+{
+    std::vector<std::vector<uint8_t>> tx_data;
+    const uint256& hash = tx.GetHash();
+    printf("- adding %s\n", hash.ToString().c_str());
+    tx_data.emplace_back(hash.begin(), hash.end());
+
+    for (unsigned int i = 0; i < tx.vout.size(); i++) {
+        const CTxOut& txout = tx.vout[i];
+        CScript::const_iterator pc = txout.scriptPubKey.begin();
+        std::vector<uint8_t> data;
+        while (pc < txout.scriptPubKey.end()) {
+            opcodetype opcode;
+            if (!txout.scriptPubKey.GetOp(pc, opcode, data)) break;
+            if (data.size() != 0) {
+                printf("- adding data: %s\n", HexStr(data).c_str());
+                tx_data.emplace_back(data.begin(), data.end());
+            }
+        }
+    }
+
+    for (const CTxIn& txin : tx.vin) {
+        tx_data.emplace_back(outpoint_data(txin.prevout));
+
+        CScript::const_iterator pc = txin.scriptSig.begin();
+        std::vector<uint8_t> data;
+        while (pc < txin.scriptSig.end()) {
+            opcodetype opcode;
+            if (!txin.scriptSig.GetOp(pc, opcode, data)) break;
+            if (data.size() != 0) {
+                printf("- adding in data %s\n", HexStr(data).c_str());
+            }
+        }
+    }
+    return tx_data;
+}
+
+void base_filter::digest_tx(const CTransaction& tx)
+{
+    insert(tx_data_vec(tx));
+}
+
+void base_filter::digest_block(const CBlock& block)
+{
+    for (const CTransactionRef& tx : block.vtx) {
+        insert(tx_data_vec(*tx));
+    }
+}
+
+void bloom_filter::insert(const std::vector<uint8_t>& data)
+{
+    filter.insert(data);
+}
+
+bool bloom_filter::contains(const std::vector<uint8_t>& data) const
+{
+    return filter.contains(data);
+}
+
+void bloom_filter::clear()
+{
+    filter.clear();
+}
+
+
+}  // namespace DigestFilter

src/blockdigest.h

@@ -0,0 +1,120 @@
+// Copyright (c) 2017 The Bitcoin Core developers
+// Distributed under the MIT software license, see the accompanying
+// file COPYING or http://www.opensource.org/licenses/mit-license.php.
+
+#ifndef _BITCOIN_SRC_BLOCKDIGEST_H_
+#define _BITCOIN_SRC_BLOCKDIGEST_H_
+
+#include <vector>
+#include <boost/filesystem.hpp>
+
+#include "bloom.h"
+#include "serialize.h"
+#include "uint256.h"
+#include "primitives/block.h"
+#include "chainparams.h"
+#include "validation.h"
+
+class COutPoint;
+class CBlock;
+class CTransaction;
+
+namespace DigestFilter {
+
+class base_filter {
+public:
+    virtual void insert(const std::vector<uint8_t>& data) = 0;
+    virtual void clear() = 0;
+    virtual bool contains(const std::vector<uint8_t>& data) const = 0;
+
+    std::vector<uint8_t> outpoint_data(const COutPoint& outpoint) const;
+    std::vector<std::vector<uint8_t>> tx_data_vec(const CTransaction& tx) const;
+
+    void digest_block(const CBlock& block);
+    void digest_tx(const CTransaction& tx);
+
+    virtual void insert(const std::vector<std::vector<uint8_t>>& data_vec) {
+        for (const std::vector<uint8_t>& d : data_vec) insert(d);
+    }
+
+    virtual bool contains(const std::vector<std::vector<uint8_t>>& data_vec) const {
+        for (const std::vector<uint8_t>& d : data_vec) if (contains(d)) return true;
+        return false;
+    }
+
+    virtual void insert(const COutPoint& outpoint) { insert(outpoint_data(outpoint)); }
+    virtual void insert(const uint256& hash)       { insert(std::vector<uint8_t>(hash.begin(), hash.end())); }
+    virtual void insert(const CTransaction& tx)    { insert(tx_data_vec(tx)); }
+
+    virtual bool contains(const COutPoint& outpoint) const { return contains(outpoint_data(outpoint)); }
+    virtual bool contains(const uint256& hash) const       { return contains(std::vector<uint8_t>(hash.begin(), hash.end())); }
+    virtual bool contains(const CTransaction& tx) const    { return contains(tx_data_vec(tx)); }
+};
+
+class bloom_filter : public base_filter {
+private:
+    CBloomFilter filter;
+public:
+    using base_filter::insert;
+    using base_filter::contains;
+
+    // TODO: obtain and use # of elements instead of using hard coded 128 value
+    bloom_filter() : filter(128, 0.000001, 0, BLOOM_UPDATE_ALL) {}
+    virtual ~bloom_filter() {}
+
+    void insert(const std::vector<uint8_t>& data) override;
+    bool contains(const std::vector<uint8_t>& data) const override;
+
+    void clear() override;
+
+    ADD_SERIALIZE_METHODS;
+
+    template <typename Stream, typename Operation>
+    inline void SerializationOp(Stream& s, Operation ser_action)
+    {
+        READWRITE(filter);
+    }
+};
+
+template <typename F>
+class digest {
+public:
+    F filter;
+    int32_t block_height;
+    uint256 block_hash;
+
+    bool contains(const std::vector<uint8_t>& data) { return filter.contains(data); }
+    bool contains(const COutPoint& outpoint)        { return filter.contains(outpoint); }
+    bool contains(const uint256& hash)              { return filter.contains(hash); }
+    bool contains(const CTransaction& tx)           { return filter.contains(tx); }
+
+    void populate(const int32_t desired_block_height)
+    {
+        CBlock block;
+        filter.clear();
+        CBlockIndex* pindex = chainActive[desired_block_height];
+        block_height = pindex->nHeight;
+        block_hash = pindex->phashBlock ? *pindex->phashBlock : uint256();
+        const Consensus::Params& consensusParams = Params().GetConsensus();
+        if (pindex) {
+            if (!ReadBlockFromDisk(block, pindex, consensusParams)) {
+                assert(!"cannot load block from disk");
+            }
+            filter.digest_block(block);
+        }
+    }
+
+    ADD_SERIALIZE_METHODS;
+
+    template <typename Stream, typename Operation>
+    inline void SerializationOp(Stream& s, Operation ser_action)
+    {
+        READWRITE(block_height);
+        READWRITE(block_hash);
+        READWRITE(filter);
+    }
+};
+
+}  // namespace DigestFilter
+
+#endif  // _BITCOIN_SRC_BLOCKDIGEST_H_

src/init.cpp

@@ -376,6 +376,7 @@ std::string HelpMessage(HelpMessageMode mode)
     strUsage += HelpMessageOpt("-onlynet=<net>", _("Only connect to nodes in network <net> (ipv4, ipv6 or onion)"));
     strUsage += HelpMessageOpt("-permitbaremultisig", strprintf(_("Relay non-P2SH multisig (default: %u)"), DEFAULT_PERMIT_BAREMULTISIG));
     strUsage += HelpMessageOpt("-peerbloomfilters", strprintf(_("Support filtering of blocks and transaction with bloom filters (default: %u)"), DEFAULT_PEERBLOOMFILTERS));
+    strUsage += HelpMessageOpt("-bfd", strprintf(_("Support block filter digests (default: %u)"), DEFAULT_BFD));
     strUsage += HelpMessageOpt("-port=<port>", strprintf(_("Listen for connections on <port> (default: %u or testnet: %u)"), Params(CBaseChainParams::MAIN).GetDefaultPort(), Params(CBaseChainParams::TESTNET).GetDefaultPort()));
     strUsage += HelpMessageOpt("-proxy=<ip:port>", _("Connect through SOCKS5 proxy"));
     strUsage += HelpMessageOpt("-proxyrandomize", strprintf(_("Randomize credentials for every proxy connection. This enables Tor stream isolation (default: %u)"), DEFAULT_PROXYRANDOMIZE));
@@ -1034,6 +1035,10 @@ bool AppInitParameterInteraction()
     if (GetBoolArg("-peerbloomfilters", DEFAULT_PEERBLOOMFILTERS))
         nLocalServices = ServiceFlags(nLocalServices | NODE_BLOOM);
 
+    if (GetBoolArg("-bfd", DEFAULT_BFD)) {
+        nLocalServices = ServiceFlags(nLocalServices | NODE_BFD);
+    }
+
     if (GetArg("-rpcserialversion", DEFAULT_RPC_SERIALIZE_VERSION) < 0)
         return InitError("rpcserialversion must be non-negative.");
 

src/net_processing.cpp

@@ -29,6 +29,7 @@
 #include "utilmoneystr.h"
 #include "utilstrencodings.h"
 #include "validationinterface.h"
+#include "blockdigest.h"
 
 #include <boost/thread.hpp>
 
@@ -1190,6 +1191,16 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr
         }
     }
 
+    if (!(pfrom->GetLocalServices() & NODE_BFD) &&
+            (strCommand == NetMsgType::GETBFD ||
+             strCommand == NetMsgType::GETBFDH))
+    {
+        fprintf(stderr, "Misbehaving node (BFD commands disabled on this node)\n");
+        LOCK(cs_main);
+        Misbehaving(pfrom->GetId(), 100);
+        return false;
+    }
+
     if (strCommand == NetMsgType::REJECT)
     {
         if (fDebug) {
@@ -2522,6 +2533,50 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr
     }
 
 
+    else if (strCommand == NetMsgType::BFD)
+    {
+        // TODO: Do something? We don't really support this in the Bitcoin Core
+        // TODO: client itself. One use case would be to sync up a node which
+        // TODO: intends to prune blocks, though.
+        LogPrint("net", "received unrequested BFD; ignoring\n");
+    }
+
+
+    else if (strCommand == NetMsgType::BFDH)
+    {
+        // TODO: Do something? We don't really support this in the Bitcoin Core
+        // TODO: client itself. One use case would be to sync up a node which
+        // TODO: intends to prune blocks, though.
+        LogPrint("net", "received unrequested BFDH; ignoring\n");
+    }
+
+
+    else if (strCommand == NetMsgType::GETBFD)
+    {
+        int32_t desiredBlockHeight;
+        vRecv >> desiredBlockHeight;
+        printf("GetBFD %d\n", desiredBlockHeight);
+
+        DigestFilter::digest<DigestFilter::bloom_filter> d;
+        d.populate(desiredBlockHeight);
+        connman.PushMessage(pfrom, msgMaker.Make(NetMsgType::BFD, d));
+    }
+
+
+    else if (strCommand == NetMsgType::GETBFDH)
+    {
+        int32_t desiredBlockHeight;
+        vRecv >> desiredBlockHeight;
+        printf("GetBFDH %d\n", desiredBlockHeight);
+
+        DigestFilter::digest<DigestFilter::bloom_filter> d;
+        d.populate(desiredBlockHeight);
+        CHashWriter ss(SER_GETHASH, 0);
+        ss << d;
+        connman.PushMessage(pfrom, msgMaker.Make(NetMsgType::BFDH, ss.GetHash()));
+    }
+
+
     else if (strCommand == NetMsgType::FILTERLOAD)
     {
         CBloomFilter filter;

src/protocol.cpp

@@ -39,6 +39,10 @@ const char *SENDCMPCT="sendcmpct";
 const char *CMPCTBLOCK="cmpctblock";
 const char *GETBLOCKTXN="getblocktxn";
 const char *BLOCKTXN="blocktxn";
+const char *GETBFD="getbfd";
+const char *BFD="bfd";
+const char *GETBFDH="getbfdh";
+const char *BFDH="bfdh";
 };
 
 /** All known message types. Keep this in the same order as the list of
@@ -71,6 +75,10 @@ const static std::string allNetMessageTypes[] = {
     NetMsgType::CMPCTBLOCK,
     NetMsgType::GETBLOCKTXN,
     NetMsgType::BLOCKTXN,
+    NetMsgType::GETBFD,
+    NetMsgType::BFD,
+    NetMsgType::GETBFDH,
+    NetMsgType::BFDH,
 };
 const static std::vector<std::string> allNetMessageTypesVec(allNetMessageTypes, allNetMessageTypes+ARRAYLEN(allNetMessageTypes));