From a8fd82c6a67b3542b49b76efa670966647ee3608 Mon Sep 17 00:00:00 2001 From: "David A. Harding" Date: Tue, 18 Nov 2014 10:02:50 -0500 Subject: [PATCH] Dev Docs: Describe Filterload Message (Final P2P Message To Document) New material: * Add documentation for `filterload` message to devref. This is the last P2P protocol message which needed documentation. * Add an example for creating a bloom filter to the devex, as well as an example of checking data against that filter. Edits: * Change "object" to "element" in previous `filteradd` text. I decided "transaction element" made more sense than the more generic "object". Text should be fully consistent across both `filterload` and `filteradd` descriptions. * Mentioned that I think the example hexdump in the `alert` section is public domain. (Only the hex is taken from the wiki; the annotation is my own work.) --- _autocrossref.yaml | 6 + _includes/example_p2p_networking.md | 241 ++++++++++++++++++++++++- _includes/ref_p2p_networking.md | 264 ++++++++++++++++++++++++++-- _includes/ref_transactions.md | 2 +- _includes/references.md | 6 + img/dev/en-bloom-update.dot | 72 ++++++++ img/dev/en-bloom-update.png | Bin 0 -> 7339 bytes img/dev/en-bloom-update.svg | 140 +++++++++++++++ 8 files changed, 713 insertions(+), 18 deletions(-) create mode 100644 img/dev/en-bloom-update.dot create mode 100644 img/dev/en-bloom-update.png create mode 100644 img/dev/en-bloom-update.svg diff --git a/_autocrossref.yaml b/_autocrossref.yaml index 1ff0046300..e4c5b75afa 100644 --- a/_autocrossref.yaml +++ b/_autocrossref.yaml @@ -63,6 +63,8 @@ confirmations: confirmed transactions: consensus: consensus rules: +data-pushing op code: +data-pushing op codes: data-pushing op code denomination: denominations: denomination DER format: der @@ -87,6 +89,8 @@ fiat: '`filteradd` messages': filteradd message '`filterclear` message': filterclear message '`filterclear` messages': filterclear message +'`filterload` message': filterload message +'`filterload` messages': filterload message fork: forks: fork genesis block: @@ -175,6 +179,8 @@ op codes: op code '`op_hash160`': op_hash160 '`op_return`': op_return '`op_verify`': op_verify +outpoint: +outpoints: outpoint outputs: output output: output index: diff --git a/_includes/example_p2p_networking.md b/_includes/example_p2p_networking.md index 1297c1dabf..60b2d25982 100644 --- a/_includes/example_p2p_networking.md +++ b/_includes/example_p2p_networking.md @@ -1,5 +1,233 @@ ## P2P Network +### Creating A Bloom Filter + +{% autocrossref %} + +In this section, we'll use variable names that correspond to the field +names in the [`filterload` message documentation][filterload message]. +Each code block precedes the paragraph describing it. + +{% highlight python %} +#!/usr/bin/env python + +BYTES_MAX = 36000 +FUNCS_MAX = 50 + +nFlags = 0 +{% endhighlight %} + +We start by setting some maximum values defined in BIP37: the maximum +number of bytes allowed in a filter and the maximum number of hash +functions used to hash each piece of data. We also set nFlags to zero, +indicating we don't want the remote node to update the filter for us. +(We won't use nFlags again in the sample program, but real programs will +need to use it.) + +{% highlight python %} +n = 1 +p = 0.0001 +{% endhighlight %} + +We define the number (n) of elements we plan to insert into the filter +and the false positive rate (p) we want to help protect our privacy. For +this example, we will set *n* to one element and *p* to a rate of +1-in-10,000 to produce a small and precise filter for illustration +purposes. In actual use, your filters will probably be much larger. + +{% highlight python %} +from math import log +nFilterBytes = int(min((-1 / log(2)**2 * n * log(p)) / 8, BYTES_MAX)) +nHashFuncs = int(min(nFilterBytes * 8 / n * log(2), FUNCS_MAX)) + +from bitarray import bitarray # from pypi.python.org/pypi/bitarray +vData = nFilterBytes * 8 * bitarray('0', endian="little") +{% endhighlight %} + +Using the formula described in BIP37, we calculate the ideal size of the +filter (in bytes) and the ideal number of hash functions to use. Both +are truncated down to the nearest whole number and both are also +constrained to the maximum values we defined earlier. The results of +this particular fixed computation are 2 filter bytes and 11 hash +functions. We then use *nFilterBytes* to create a little-endian bit +array of the appropriate size. + +{% highlight python %} +nTweak = 0 +{% endhighlight %} + +We also should choose a value for *nTweak*. In this case, we'll simply +use zero. + +{% highlight python %} +import pyhash # from https://github.com/flier/pyfasthash +murmur3 = pyhash.murmur3_32() + +def bloom_hash(nHashNum, data): + seed = (nHashNum * 0xfba4c795 + nTweak) & 0xffffffff + return( murmur3(data, seed=seed) % (nFilterBytes * 8) ) +{% endhighlight %} + +We setup our hash function template using the formula and 0xfba4c795 +constant set in BIP37. Note that we limit the size of the seed to four +bytes and that we're returning the result of the hash modulo the size of +the filter in bits. + +{% highlight python %} +data_to_hash = "019f5b01d4195ecbc9398fbf3c3b1fa9" \ + + "bb3183301d7a1fb3bd174fcfa40a2b65" +data_to_hash = data_to_hash.decode("hex") +{% endhighlight %} + +For the data to add to the filter, we're adding a TXID. Note that the +TXID is in internal byte order. + +{% highlight python %} +print " Filter (As Bits)" +print "nHashNum nIndex Filter 0123456789abcdef" +print "~~~~~~~~ ~~~~~~ ~~~~~~ ~~~~~~~~~~~~~~~~" +for nHashNum in range(nHashFuncs): + nIndex = bloom_hash(nHashNum, data_to_hash) + + ## Set the bit at nIndex to 1 + vData[nIndex] = True + + ## Debug: print current state + print ' {0:2} {1:2} {2} {3}'.format( + nHashNum, + hex(int(nIndex)), + vData.tobytes().encode("hex"), + vData.to01() + ) + +print +print "Bloom filter:", vData.tobytes().encode("hex") +{% endhighlight %} + +Now we use the hash function template to run a slightly different hash +function for *nHashFuncs* times. The result of each function being run +on the transaction is used as an index number: the bit at that index is +set to 1. We can see this in the printed debugging output: + +{% highlight text %} + Filter (As Bits) +nHashNum nIndex Filter 0123456789abcdef +~~~~~~~~ ~~~~~~ ~~~~~~ ~~~~~~~~~~~~~~~~ + 0 0x7 8000 0000000100000000 + 1 0x9 8002 0000000101000000 + 2 0xa 8006 0000000101100000 + 3 0x2 8406 0010000101100000 + 4 0xb 840e 0010000101110000 + 5 0x5 a40e 0010010101110000 + 6 0x0 a50e 1010010101110000 + 7 0x8 a50f 1010010111110000 + 8 0x5 a50f 1010010111110000 + 9 0x8 a50f 1010010111110000 + 10 0x4 b50f 1010110111110000 + +Bloom filter: b50f +{% endhighlight %} + +Notice that in iterations 8 and 9, the filter did not change because the +corresponding bit was already set in a previous iteration (5 and 7, +respectively). This is a normal part of bloom filter operation. + +We only added one element to the filter above, but we could repeat the +process with additional elements and continue to add them to the same +filter. (To maintain the same false-positive rate, you would need a +larger filter size as computed earlier.) + +Note: for a more optimized Python implementation with fewer external +dependencies, see [python-bitcoinlib's][python-bitcoinlib] bloom filter +module which is based directly on Bitcoin Core's C++ implementation. + +Using the `filterload` message format, the complete filter created above +would be the binary form of the annotated hexdump shown below: + +{% highlight text %} +02 ......... Filter bytes: 2 +b50f ....... Filter: 1010 1101 1111 0000 +0b000000 ... nHashFuncs: 11 +00000000 ... nTweak: 0/none +00 ......... nFlags: BLOOM_UPDATE_NONE +{% endhighlight %} + +{% endautocrossref %} + +### Evaluating A Bloom Filter + +{% autocrossref %} + +Using a bloom filter to find matching data is nearly identical to +constructing a bloom filter---except that at each step we check to see +if the calculated index bit is set in the existing filter. + +{% highlight python %} +vData = bitarray(endian='little') +vData.frombytes("b50f".decode("hex")) +nHashFuncs = 11 +nTweak = 0 +nFlags = 0 +{% endhighlight %} + +Using the bloom filter created above, we import its various parameters. +Note, as indicated in the section above, we won't actually use *nFlags* +to update the filter. + +{% highlight python %} +def contains(nHashFuncs, data_to_hash): + for nHashNum in range(nHashFuncs): + ## bloom_hash as defined in previous section + nIndex = bloom_hash(nHashNum, data_to_hash) + + if vData[nIndex] != True: + print "MATCH FAILURE: Index {0} not set in {1}".format( + hex(int(nIndex)), + vData.to01() + ) + return False +{% endhighlight %} + +We define a function to check an element against the provided filter. +When checking whether the filter might contain an element, we test to +see whether a particular bit in the filter is already set to 1 (if it +isn't, the match fails). + +{% highlight python %} +## Test 1: Same TXID as previously added to filter +data_to_hash = "019f5b01d4195ecbc9398fbf3c3b1fa9" \ + + "bb3183301d7a1fb3bd174fcfa40a2b65" +data_to_hash = data_to_hash.decode("hex") +contains(nHashFuncs, data_to_hash) +{% endhighlight %} + +Testing the filter against the data element we previously added, we get +no output (indicating a possible match). Recall that bloom filters have +a zero false negative rate---so they should always match the inserted +elements. + +{% highlight python %} +## Test 2: Arbitrary string +data_to_hash = "1/10,000 chance this ASCII string will match" +contains(nHashFuncs, data_to_hash) +{% endhighlight %} + +Testing the filter against an arbitrary element, we get the failure +output below. Note: we created the filter with a 1-in-10,000 false +positive rate (which was rounded up somewhat when we truncated), so it +was possible this arbitrary string would've matched the filter anyway. +It is not possible to set a bloom filter to a false positive rate of +zero, so your program will always have to deal with false positives. +The output below shows us that one of the hash functions returned an +index number of 0x06, but that bit wasn't set in the filter, causing the +match failure: + +{% highlight text %} +MATCH FAILURE: Index 0x6 not set in 1010110111110000 +{% endhighlight %} + +{% endautocrossref %} + ### Retrieving A MerkleBlock {% autocrossref %} @@ -77,12 +305,17 @@ script, but we will sleep a short bit and send back our own `verack` message as if we had accepted their `version` message. {% highlight python %} -send("filterload", "02b50f0b0000000000000000") +send("filterload", + "02" ........ Filter bytes: 2 + "b50f" ....... Filter: 1010 1101 1111 0000 + "0b000000" ... nHashFuncs: 11 + "00000000" ... nTweak: 0/none + "00" ......... nFlags: BLOOM_UPDATE_NONE +) {% endhighlight %} -We set a bloom filter with the `filterload` message. This filter was -quickly created using [python-bitcoinlib][]'s bloom module. +We set a bloom filter with the `filterload` message. This filter is +described in the two preceeding sections. {% highlight python %} send("getdata", diff --git a/_includes/ref_p2p_networking.md b/_includes/ref_p2p_networking.md index 2327a3909d..f54516e64c 100644 --- a/_includes/ref_p2p_networking.md +++ b/_includes/ref_p2p_networking.md @@ -694,7 +694,9 @@ introduced in protocol version 311. The annotated hexdump below shows an `alert` message. (The message header has been omitted.) - {% highlight text %} @@ -746,21 +748,24 @@ alert.cpp] source code for the parameters of this message. *Added in protocol version 70001 as described by BIP37.* -The `filteradd` message tells the receiving peer to add a single object to -a previously-set bloom filter, such as a new public key. The object is +The `filteradd` message tells the receiving peer to add a single element to +a previously-set bloom filter, such as a new public key. The element is sent directly to the receiving peer; the peer then uses the parameters set -in the `filterload` message to add the object to the bloom filter. +in the `filterload` message to add the element to the bloom filter. -Because the object is sent directly to the receiving peer, there is no -obfuscation of the object and none of the plausible-deniability privacy +Because the element is sent directly to the receiving peer, there is no +obfuscation of the element and none of the plausible-deniability privacy provided by the bloom filter. Clients that want to maintain greater privacy should recalculate the bloom filter themselves and send a new `filterload` message with the recalculated bloom filter. -| Bytes | Name | Data Type | Description -|----------|--------------|------------------|----------------- -| *Varies* | object bytes | compactSize uint | The number of bytes in the following object field. -| *Varies* | object | uint8_t[] | The object to add to the current filter. Maximum of 520 bytes, which is the maximum size of an object which can be pushed onto the stack in a pubkey or signature script. Objects must be sent in the byte order they would use when appearing in a raw transaction; for example, hashes should be sent in internal byte order. +| Bytes | Name | Data Type | Description +|----------|---------------|------------------|----------------- +| *Varies* | element bytes | compactSize uint | The number of bytes in the following element field. +| *Varies* | element | uint8_t[] | The element to add to the current filter. Maximum of 520 bytes, which is the maximum size of an element which can be pushed onto the stack in a pubkey or signature script. Elements must be sent in the byte order they would use when appearing in a raw transaction; for example, hashes should be sent in internal byte order. + +Note: a `filteradd` message will not be accepted unless a filter was +previously set with the `filterload` message. The annotated hexdump below shows a `filteradd` message adding a TXID. (The message header has been omitted.) This TXID appears in the same @@ -769,9 +774,9 @@ block used for the example hexdump in the `merkleblock` message; if that six hashes are returned instead of four. {% highlight text %} -20 ................................. Object bytes: 32 +20 ................................. Element bytes: 32 fdacf9b3eb077412e7a968d2e4f11b9a -9dee312d666187ed77ee7d26af16cb0b ... Object (A TXID) +9dee312d666187ed77ee7d26af16cb0b ... Element (A TXID) {% endhighlight %} {% endautocrossref %} @@ -797,9 +802,242 @@ section][message header] for an example of a message without a payload. {% endautocrossref %} - +#### FilterLoad + +{% autocrossref %} + +*Added in protocol version 70001 as described by BIP37.* + +The `filterload` message tells the receiving peer to filter all relayed +transactions and requested merkleblocks through the provided filter. +This allows clients to receive transactions relevant to their wallet +plus a configurable rate of false positive transactions which can +provide plausible-deniability privacy. + +| Bytes | Name | Data Type | Description +|----------|--------------|-----------|--------------- +| *Varies* | nFilterBytes | uint8_t[] | Number of bytes in the following filter bit field. +| *Varies* | filter | uint8_t[] | A bit field of arbitrary byte-aligned size. The maximum size is 36,000 bytes. +| 4 | nHashFuncs | uint32_t | The number of hash functions to use in this filter. The maximum value allowed in this field is 50. +| 4 | nTweak | uint32_t | A arbitrary value to add to the seed value in the hash function used by the bloom filter. +| 1 | nFlags | uint8_t | A set of flags that control how outpoints corresponding to a matched pubkey script are are added to the filter. See the table in the Updating A Bloom Filter subsection below. + +The annotated hexdump below shows a `filterload` message. (The message +header has been omitted.) For an example of how this payload was +created, see the [filterload example][section creating a bloom filter]. + +{% highlight text %} +02 ......... Filter bytes: 2 +b50f ....... Filter: 1010 1101 1111 0000 +0b000000 ... nHashFuncs: 11 +00000000 ... nTweak: 0/none +00 ......... nFlags: BLOOM_UPDATE_NONE +{% endhighlight %} + +**Initializing A Bloom Filter** + +Filters have two core parameters: the size of the bit field and the +number of hash functions to run against each data element. The following +formulas from BIP37 will allow you to automatically select appropriate +values based on the number of elements you plan to insert into the +filter (*n*) and the false positive rate (*p*) you desire to maintain +plausible deniability. + +* Size of the bit field in bytes (*nFilterBytes*), up to a maximum of + 36,000: `(-1 / log(2)**2 * n * log(p)) / 8` + +* Hash functions to use (*nHashFuncs*), up to a maximum of 50: + `nFilterBytes * 8 / n * log(2)` + +Note that the filter matches parts of transactions (transaction +elements), so the false positive rate is relative to the number of +elements checked---not the number of transactions checked. Each normal +transaction has a minimum of four matchable elements (described in the +comparison subsection below), so a filter with a false-positive rate of +1 percent will match about 4 percent of all transactions at a minimum. +According to BIP37, the formulas and limits described above provide +support for bloom filters containing 20,000 items with a false positive +rate of less than 0.1 percent or 10,000 items with a false positive rate +of less than 0.0001 percent. +Once the size of the bit field is known, the bit field should be +initialized as all zeroes. + +**Populating A Bloom Filter** + +The bloom filter is populated using between 1 and 50 unique hash +functions (the number specified per filter by the *nHashFuncs* +field). Instead of using up to 50 different hash function +implementations, a single implementation is used with a unique seed +value for each function. + +The seed is `nHashNum * 0xfba4c795 + nTweak` as a *uint32\_t*, where the values +are: + +* **nHashNum** is the sequence number for this hash + function, starting at 0 for the first hash iteration and increasing up + to the value of the *nHashFuncs* field (minus one) for the last hash + iteration. + +* **0xfba4c795** is a constant optimized to create large differences in + the seed for different values of *nHashNum*. + +* **nTweak** is a per-filter constant set by the client to require the use + of an arbitrary set of hash functions. + +If the seed resulting from the formula above is larger than four bytes, +it must be truncated to its four most significant bytes (for example, +`0x8967452301 & 0xffffffff → 0x67452301`). + +The actual hash function implementation used is the [32-bit Murmur3 hash +function][murmur3]. + +![Warning icon](/img/icon_warning.svg) +**Warning:** the Murmur3 hash function has separate 32-bit and 64-bit +versions that produce different results for the same input. Only the +32-bit Murmur3 version is used with Bitcoin bloom filters. + +The data to be hashed can be any transaction element which the bloom +filter can match. See the next subsection for the list of transaction +elements checked against the filter. The largest element which can be +matched is a script data push of 520 bytes, so the data should never +exceed 520 bytes. + +The example below from Bitcoin Core [bloom.cpp][core bloom.cpp hash] combines +all the steps above to create the hash function template. The seed is +the first parameter; the data to be hashed is the second parameter. The +result is a uint32\_t modulo the size of the bit field in bits. + +{% highlight c++ %} +MurmurHash3(nHashNum * 0xFBA4C795 + nTweak, vDataToHash) % (vData.size() * 8) +{% endhighlight %} + +Each data element to be added to the filter is hashed by *nHashFuncs* +number of hash functions. Each time a hash function is run, the result +will be the index number (*nIndex*) of a bit in the bit field. That bit +must be set to 1. For example if the filter bit field was `00000000` and +the result is 5, the revised filter bit field is `00000100` (the first bit +is bit 0). + +It is expected that sometimes the same index number will be returned +more than once when populating the bit field; this does not affect the +algorithm---after a bit is set to 1, it is never changed back to 0. + +After all data elements have been added to the filter, each set of eight +bits is converted into a little-endian byte. These bytes are the value +of the *filter* field. + +**Comparing Transaction Elements To A Bloom Filter** + +To compare an arbitrary data element against the bloom filter, it is +hashed using the same parameters used to create the bloom filter. +Specifically, it is hashed *nHashFuncs* times, each time using the same +*nTweak* provided in the filter, and the resulting output is modulo the +size of the bit field provided in the *filter* field. After each hash is +performed, the filter is checked to see if the bit at that indexed +location is set. For example if the result of a hash is `5` and the +filter is `01001110`, the bit is considered set. + +If the result of every hash points to a set bit, the filter matches. If +any of the results points to an unset bit, the filter does not match. + +The following transaction elements are compared against bloom filters. +All elements will be hashed in the byte order used in blocks (for +example, TXIDs will be in internal byte order). + +* **TXIDs:** the transaction's SHA256(SHA256()) hash. + +* **Outpoints:** each 36-byte outpoint used this transaction's input + section is individually compared to the filter. + +* **Signature Script Data:** each element pushed onto the stack by a + data-pushing op code in a signature script from this transaction is + individually compared to the filter. This includes data elements + present in P2SH redeem scripts when they are being spent. + +* **PubKey Script Data:** each element pushed onto the the stack by a + data-pushing op code in any pubkey script from this transaction is + individually compared to the filter. (If a pubkey script element + matches the filter, the filter will be immediately updated if the + `BLOOM_UPDATE_ALL` flag was set; if the pubkey script is in the P2PKH + format and matches the filter, the filter will be immediately updated + if the `BLOOM_UPDATE_P2PUBKEY_ONLY` flag was set. See the subsection + below for details.) + +The following annotated hexdump of a transaction is from the [raw +transaction format section][raw transaction format]; the elements which +would be checked by the filter are emphasized in bold. Note that this +transaction's TXID (**`01000000017b1eab[...]`**) would also be checked, +and that the outpoint TXID and index number below would be checked as a +single 36-byte element. + +
01000000 ................................... Version
+
+01 ......................................... Number of inputs
+|
+| 7b1eabe0209b1fe794124575ef807057
+| c77ada2138ae4fa8d6c4de0398a14f3f ......... Outpoint TXID
+| 00000000 ................................. Outpoint index number
+|
+| 49 ....................................... Bytes in sig. script: 73
+| | 48 ..................................... Push 72 bytes as data
+| | | 30450221008949f0cb400094ad2b5eb3
+| | | 99d59d01c14d73d8fe6e96df1a7150de
+| | | b388ab8935022079656090d7f6bac4c9
+| | | a94e0aad311a4268e082a725f8aeae05
+| | | 73fb12ff866a5f01 ..................... Secp256k1 signature
+|
+| ffffffff ................................. Sequence number: UINT32_MAX
+
+01 ......................................... Number of outputs
+| f0ca052a01000000 ......................... Satoshis (49.99990000 BTC)
+|
+| 19 ....................................... Bytes in pubkey script: 25
+| | 76 ..................................... OP_DUP
+| | a9 ..................................... OP_HASH160
+| | 14 ..................................... Push 20 bytes as data
+| | | cbc20a7664f2f69e5355aa427045bc15
+| | | e7c6c772 ............................. PubKey hash
+| | 88 ..................................... OP_EQUALVERIFY
+| | ac ..................................... OP_CHECKSIG
+
+00000000 ................................... locktime: 0 (a block height)
+
+ +**Updating A Bloom Filter** + +Clients will often want to track inputs that spend outputs (outpoints) +relevant to their wallet, so the filterload field *nFlags* can be set to +allow the filtering node to update the filter when a match is found. +When the filtering node sees a pubkey script that pays a pubkey, +address, or other data element matching the filter, the filtering node +immediately updates the filter with the outpoint corresponding to that +pubkey script. + +![Automatically Updating Bloom Filters](/img/dev/en-bloom-update.svg) + +If an input later spends that outpoint, the filter will match it, +allowing the filtering node to tell the client that one of its +transaction outputs has been spent. + +The *nFlags* field has three allowed values: + +| Value | Name | Description +|-------|----------------------------|--------------- +| 0 | BLOOM_UPDATE_NONE | The filtering node should not update the filter. +| 1 | BLOOM_UPDATE_ALL | If the filter matches any data element in a pubkey script, the corresponding outpoint is added to the filter. +| 2 | BLOOM_UPDATE_P2PUBKEY_ONLY | If the filter matches any data element in a pubkey script and that script is either a P2PKH or non-P2SH pay-to-multisig script, the corresponding outpoint is added to the filter. + +In addition, because the filter size stays the same even though +additional elements are being added to it, the false positive rate +increases. Each false positive can result in another element being added +to the filter, creating a feedback loop that can (after a certain point) +make the filter useless. For this reason, clients using automatic filter +updates need to monitor the actual false positive rate and send a new +filter when the rate gets too high. + +{% endautocrossref %} #### GetAddr diff --git a/_includes/ref_transactions.md b/_includes/ref_transactions.md index 06256f6aec..17b6e7f1bb 100644 --- a/_includes/ref_transactions.md +++ b/_includes/ref_transactions.md @@ -238,7 +238,7 @@ Each non-coinbase input spends an outpoint from a previous transaction. {% endautocrossref %} -**Outpoint: The Specific Part Of A Specific Output** +**[Outpoint][]{:#term-outpoint}{:.term}: The Specific Part Of A Specific Output** {% autocrossref %} diff --git a/_includes/references.md b/_includes/references.md index 19a18df5f9..7159b4ccde 100644 --- a/_includes/references.md +++ b/_includes/references.md @@ -37,6 +37,7 @@ [confirmations]: /en/developer-guide#term-confirmation "The number of blocks which would need to be modified to remove or modify a transaction" [consensus]: /en/developer-guide#term-consensus "When several nodes (usually most nodes on the network) all have the same blocks in their locally-validated block chain." [consensus rules]: /en/developer-guide#term-consensus-rules "The block validation rules that full nodes follow to stay in consensus with other nodes." +[data-pushing op code]: https://en.bitcoin.it/wiki/Script#Constants "Any op code from 0x01 to 0x4e which pushes data on to the script evaluation stack" [denomination]: /en/developer-guide#term-denomination "bitcoins (BTC), bitcents (cBTC), millibitcoins (mBTC), bits (uBTC, microbitcoins), or satoshis" [difficulty]: /en/developer-guide#term-difficulty "A number corresponding to the target threshold which indicates how difficult it will be to find the next block" [dns seed]: /en/developer-guide#term-dns-seed "A DNS server which returns IP addresses of full nodes on the Bitcoin network to assist in peer discovery." @@ -48,6 +49,7 @@ [fiat]: /en/developer-guide#term-fiat "National currencies such as the dollar or euro" [filteradd message]: /en/developer-reference#filteradd "A P2P protocol message used to add a data element to an existing bloom filter." [filterclear message]: /en/developer-reference#filterclear "A P2P protocol message used to remove an existing bloom filter." +[filterload message]: /en/developer-reference#filterclear "A P2P protocol message used send a filter to a remote peer, requesting that they only send transactions which match the filter." [fork]: /en/developer-guide#term-fork "When two or more blocks have the same block height, forking the block chain." [genesis block]: /en/developer-guide#term-genesis-block "The first block created; also called block 0" [getaddr message]: /en/developer-reference#getaddr "A P2P protool message used to request an addr message containing connection information for other nodes" @@ -105,6 +107,7 @@ [op_hash160]: /en/developer-reference#term-op-hash160 "Operation which converts the entry below it on the stack into a RIPEMD(SHA256()) hashed version of itself" [op_return]: /en/developer-reference#term-op-return "Operation which terminates the script in failure" [op_verify]: /en/developer-reference#term-op-verify "Operation which terminates the script if the entry below it on the stack is non-true (zero)" +[outpoint]: /en/developer-reference#term-outpoint "The structure used to refer to a particular transaction output, considing of a 32-byte TXID and a 4-byte output index number (vout)." [output]: /en/developer-guide#term-output "The output of a transaction which transfers value to a pubkey script" [output index]: /en/developer-guide#term-output-index "The sequentially-numbered index of outputs in a single transaction starting from 0" [P2PKH]: /en/developer-guide#term-p2pkh "A pubkey script which Pays To PubKey Hashes (P2PKH), allowing spending of satoshis to anyone with a Bitcoin address" @@ -295,6 +298,7 @@ [raw transaction format]: /en/developer-reference#raw-transaction-format [RPC]: /en/developer-reference#remote-procedure-calls-rpcs [RPCs]: /en/developer-reference#remote-procedure-calls-rpcs +[section creating a bloom filter]: /en/developer-examples#creating-a-bloom-filter [section detecting forks]: /en/developer-guide#detecting-forks [section getblocktemplate]: /en/developer-guide#getblocktemplate-rpc [section hash byte order]: /en/developer-reference#hash-byte-order @@ -370,6 +374,7 @@ [irc channels]: https://en.bitcoin.it/wiki/IRC_channels [libblkmaker]: https://gitorious.org/bitcoin/libblkmaker [makeseeds script]: https://github.com/bitcoin/bitcoin/tree/master/contrib/seeds +[murmur3]: https://en.wikipedia.org/wiki/MurmurHash [man-in-the-middle]: https://en.wikipedia.org/wiki/Man-in-the-middle_attack [MIME]: https://en.wikipedia.org/wiki/Internet_media_type [mozrootstore]: https://www.mozilla.org/en-US/about/governance/policies/security-group/certs/ @@ -391,4 +396,5 @@ +[core bloom.cpp hash]: https://github.com/bitcoin/bitcoin/blob/cbf28c6619fe348a258dfd7d08bdbd2392d07511/src/bloom.cpp#L46 [MAX_SIZE]: https://github.com/bitcoin/bitcoin/blob/60abd463ac2eaa8bc1d616d8c07880dc53d97211/src/serialize.h#L23 diff --git a/img/dev/en-bloom-update.dot b/img/dev/en-bloom-update.dot new file mode 100644 index 0000000000..57d00d2419 --- /dev/null +++ b/img/dev/en-bloom-update.dot @@ -0,0 +1,72 @@ +digraph { + +size="6.25"; +rankdir=TB; +nodesep=1.05; +ranksep=0.2; +splines="false" + +edge [ penwidth = 1.75, fontname="Sans" ] +node [ penwidth = 1.75, shape = "box", fontname="Sans", ] +graph [ penwidth = 1.75, fontname="Sans" ] + +subgraph cluster_client { + graph [ penwidth = 0 ]; + subgraph cluster_client1 { + graph [ penwidth = 0 ]; + + address [ label = "Address To Match" ]; + filter1 [ label = "Filter That\n\ \ Matches Address\ \ ", style = "diagonals" ]; + mymatch1 [ label = "Transaction 1", style = "invis" ]; + mymatch2 [ label = "Transaction 2", style = "invis" ]; + + address -> filter1; + filter1 -> mymatch1 [ style = "invis" ]; + mymatch1 -> mymatch2 [ style = "invis" ]; + + } + + label = "Client" +} + +filter1 -> filter2 [ constraint = false ]; + +subgraph cluster_node { + graph [ penwidth = 0 ]; + subgraph cluster_node2 { + graph [ penwidth = 0 ]; + + tx2 [ label = "Transaction 2" ]; + filter3 [ label = "Filter Updated\n\ \ \ \ With Outpoint \ ", style = "diagonals" ]; + match2 [ label = "Transaction 2\nSpends Outpoint", shape = "none" ]; + + tx2 -> filter3; + filter3 -> match2 [ minlen = 2 ]; + } + + + subgraph cluster_node1 { + graph [ penwidth = 0 ]; + + tx1 [ label = "Transaction 1" ]; + filter2 [ label = "Filter That\n\ \ Matches Address\ \ ", style = "diagonals" ]; + match1 [ label = "Transaction 1\nPays Address", shape = "none" ] + + tx1 -> filter2; + filter2 -> match1; + } + + match1 -> mymatch1 [ constraint = false ]; + + filter2 -> filter3 [constraint = false ]; + match1 -> filter3 [ constraint = false ]; + + + match2 -> mymatch2 [ constraint = false ]; + + label = "Full Node" +} + +label = "Automatically Updating Bloom Filters To Track Relevant Transactions" + +} diff --git a/img/dev/en-bloom-update.png b/img/dev/en-bloom-update.png new file mode 100644 index 0000000000000000000000000000000000000000..320410b71a7b78f3d4c6ba8ad6afc36cb1c6791c GIT binary patch literal 7339 zcmb7Jby!qiw;nnK=}u`J1f&H8r5mYnK!G8KMoK~u7(zNkx?vDO7+^?ADd|Q)YLF0t zp*tk*px^J_=ef^!?{n{8XYX~^Ua{YO_TK9~5l^*MNQoJV0RRB0nyR860Duh!0I=8z zF#>>AqTYSX1KUDFMG&b!L+k^IqOhyhw|q68h@iIo8j(rLIDto;zL{k z{HRVy1-$@mRGYP`gU)-@(@VFP(MjwZ#s7u|2AjWtVX*lN7^LDaV3^)tz(_za`#GYC zi`4WMFTVQmMThfF*El1#{4agXbyM8vjj0Hxc_DzNRJPDlWr zm%JxL?53C!UtZ!Oz@F)6bFm3*C~?kYh!b0|s}PPd_U$uG@t0v~G0QchLxFP1aWh0# z+qZR*P;A(q5GC^ZpUI{{V>6-N-n3W>bW=fX!A#R&UA4;AZl{gO}y-KsaX3Baff3eyJh;1r%Na)fjW zo$2={^h8oWMOp9%9GU45*8sGxOyC@m8LeX-tA4hg%}dvQX$1!?(Ff`ltG&LpSL zNm^)pOu_)nuU^Ms(l-)=STFB735l}G7**`rmfOQ%cZS7s96)JL?>bPC(E{?H8Amw# zrj0%u)e5_2drcYi#}t#O(BTi7{#jBc8*IQwClN}Q9-C2EUzh7_w&?8dew8g;5wEao zy1vR%ClAR_ntUa^lk0Rs5jYyheS8!z4a}4zO?+1}#*D^-CJHc-IfyzCY(}m7A@1&t z);eXz!xUtr@C_tlSHm8tvAt zB7-GTeR*NA(mPO`{kC=26#is!W>Mkd%FeWlu zCtFca#B9fZ2R|dHNGcpsQqyOpfR?M1ivA`JFWS+6M2T44kEq<%$a6RuRZR5aQW_xP zfX^*sWeQ?gI&X4q;^mRn18l^lSB?35nOiN#_qToH^-(uR! zOEGcyQfryyjdI*B26AqVi!*R?N_>b}(v$i4(yp576^#jVX?z+5EGZ#hBw|HZ?Sop+ z5HR|%XFodgV5R@fb9pa`1P$@537I8?z2CWtk5yiNj!&AMMqMnge$Dq`ifLgcPUd8F zzF}K-UP`1@Poy*Jy{M^+`d{H>e$tldb6e>9Dq|q|iRN3B~7q-LEROrWt35H+DMA zsonR?9%igRV9^mh`DP-!ho0nXjf^ZQI(*e-;7oGUw6%H<6#@?{_8EQ^>50Tfe07Ds zhXTPw%j%r{Amq|8jf>piff50T{zW{gcu)eA7o6v=ghV|pQF(`Ho4C-tJmEW(Od+b| zzy}(X@kv0`Djji4WEIQ!cp!__?ET6}>E%d^TghBNJ!HeG`m6=wTfMMnb>Z&n%6jxC zQ>M0JHsdCsfkKv!7Ii4D(QzP6k0GBZHt~!Zx-TN2S0sHkiW)&a2_)W%%M8i}Cbksi zSZN;3;DatlmOd*;jIaE7^EFi4S0tx~48gu(s48lK0|qM}yrBr9X5uLK@3N52yY3T38iU1c7!;m5!Bpoo|%ySu=iW7>uE;o-K}gO z5WHk|_ksLss6Yg{JbTF$D>5?J#O$pYXS#-tLh$PBf0>(nr`p;F$NRFq+SW8B?AT+4 z{R{uN>){&Hx6gG7&2<;D)7X>liB(kQ^dhZ#y%Y@+R$TIHAl~v18e4!3y#};Zh$uX; zhG6Q^*B)YXS9tV>GkPDTVHA%5=lg}*cE0b$?V&q-9|A#5$l{_P-y#ih6e}%0{Upp{ zs^pR<3`EHLf)MQq@k6R0CV;mnfbs#3!GESYxn}D>Mua1p?As#6VAjlj3OjgnJWn9d)0m4PKhAmJZ z&|H)~0I5Gt_iqr!O!GSfh=&NL9Q{#PaCkBhz+ zw*i20D^$!Y3U;{Fs~I~|e@BBOhpY;x1p24?f@#Nbj&lf)g0h86i?OAI%GP z)ir=Ldr1i(m*7RM=ik-^+lci?@q9jtg5~<^c753jD8U_BIX_&J+teR|rxQ^&ii2VN zI3PS-{~V^;8d0zW>}V}CT%0!}yl2{GeC{XK#sx+E@9B}8@bG=Nogz2hIn_7{i$Qc>w8GX!flf*84oKGl^l*1t4t0 z4M5a&glK~(_l#_xgN+-IT63**-FQ(FRkskaeR9h&eYf|cij2kgTKANcUEW77@n0*= z#@8>kuFG|HtqbG$j*AZ3AC{N1(D)*;F9;Rdb#Ox!TN4mMB?O|>mdKfYxpu#Vmbamm zp0>J-%2LdLL{gj}xG-%0{vOZc)sJFh*f z#{nQy!^4yR%;cNha%WbkeWVtobbB&R_|AnQL>2d8?|G#Uxx(Pt_Zh6onzO8}^y4^< zp%)<(P90a~Q01JR_d_57iX3j{2logF;BMRUHnP2^$zi^^iogiJ|+^SE1?MQnC&N z;2e+W^=O2K#6V}jBmWBX4hq2J))J$`?FC1?^Z|7a+5Sq4-eqj~;E-RukIKHTHxcSh zZ55M5a(M9s)duGFI8+YzemU!I?Hd4?;;ZFKtL-J+)E&k>{Vo|RQk~j0U2PFFFf$Xw zX0Cb~(fZEz>14_T4v9GFu*dX)?phzqJoQDX$K@4WhTC@rK@;E0w)2zM;#ykbt6Ky- z4_~1dj;y&Ua^4a=JUU8Xt%j|zIZgVDD$ze_7>W~c;t-%P-0GiG67M=Qq#i!#$jmn* zfZeTl>d0h+dt79LeX+r$lJ;WI%$y25TUPm=W=(~j@&;VS#3QIjfPz+syVs0I%u)AH zk94|WoPv=GX$ocwy3X!s`FOk~vyua_zbR_$Zr1RwAm2Tcg&Noj-Aw?cQHM*Y90acI z{lSxI&pm&1F~W>zO6S||T-LEklb1T^EAg52i{FFQg8R;IiFh4u)J{u<7p*XPqwH2c z%g^@0UFss2JQad^jWtis9h*@uXmrSX?z+kY&tR05uG9-9cQywW)Lb^C%&|Ex>1pK^ zXI&&QqC@6t)Em?o2fZ#dUwfhC5iAyHrB=4t{3QKxCG|7iz#3plU2EC~>m$qgk)=u8 z_zC|6OBOgD(ACOM-wk?@brKQ}b)lgFuew2V&8$5m_@)`0D@n@W7izputQcNQUnXb= z^@!9*;tTxP)=+%xxUKNLIOo-pW^%7#2n5P_=EX0=yX__&eQ;7qB(Tz+7Eci%cg`ox zfm34ntcEvbXqX2+XI`@_W#HS)4~$i=9IS#WV^3yTN3JtOsEWxR&>kG!XKFpKnx0#A zj;_}KKKhn|X-wJG;}rFyW${=9KKE%t4#=T$8Y>7^W?5RJ#h=i%I4 zX0OAlySGW|0g!e(0RE^bKxJoANf*5OSoV_Fueod+%sjk4YxwA!l7X0S$4eSVkpuq> z!Pg83uF>_(3wV1w9AUFeXh@{HxXJ6e?#g4cX`j=kPK(-Phl4*3jOF=Qc=GE8vKiDy zVx*U4@4{`Z2ZpPBJ2v&;L1_~e!5A@57LW@bWY*1CECRnsn`pGS;=C7}BzA6(uj{|L zP0KbX+?d(39)V*(0@7O_ly@75(E1j(bc-f9>BZfAxZKna^df0he?;Y@37O@FL4x3< zhsI+ibsh7xagL`JZ(+sa^Val;pQDYjzK)Jt!8X4P$&Zc^Z)(ITFk-H;-OrBpMQt#& z0lY~VtoGr$DnCB%^FRrEUpi9E_33@^r;nD;F+-ryCgNMicX_6wrB3*)h zN8ha0ZU-k3OMijR_YC|D#AJi#SGlrbDcXDAbGp73YAbV%i}AETD=eY( zngldJbcgsJ$|1bN+G%6QvgON4hgJ0;gnJlGWq-taA3*|YYNhlIFTPZrNY1jL(>1y) z!6#moTHGvsbomHVCG?Zsl$IyY44->OnkY^P7Pxg7dPL8Z=>PbU&}MVVFpmJkpcy5DEj@dbi{XzFs;`*xy{!!!vZ+#> zbXuFj(3luWs?Kr+ys79t2$JXG#`!23!;4MSJMjy;K4#CDW??Fy32v~PoL;`-CP%R+ zKMhkNV}_5tuZR8myY?nv7o=0)Cg6bUUA$zf^|t76sLabCN{~z}I4PSW*Zi2Sp_l^7YKsqaYHL@yG#H;q z8pe^*^Kb8ZI6?w44(84;^Ou3+q&|1D`Kx5#v?bx%(LoyRdwZw;`PKl=Af$jFtjO&i z%c%QIh*ZYwx6f26L3*3|a-4i>eV$)_G6`iP)oj9z$Q@}gPGg!|h&8+aSQNZ|J2|qU zJu1lttMR5QP_Q9U_(cUDxQ(6?E)K>Zv&ok9#5oYt_=k@9L!j27%<#b2og7QOFga)pUs*4MqF`1%4E?{EZ+--!-hwVjAIm z$Z<{*0JNGe^}5A>5=KJX?@5|FXN^&61RXap-}G5rA9h13EOH%Wp-Axk5iDUaf|chV zh7Us+O_N~K>W^78``h$rWXL6Cd@#OhPR~7MomQN1(MSZDzd8P@YDUJ zMjm{jwO6n)^6@-_wRhi{l*AtS!_#FL8LfZ&g7FA&0CDEl27Y9^o#kVTJsvt#`D)@T zPB)#M+T#>o^hUXnISAxq#}V8q@|WMh7so1nFe3TtrK_cFDTu{7$idJEoz?amlm3%P zGT03M3`E6oO+X{d6bRN9TJSjMt@F2R{c-yIPQQF)1!*;CY1Z_kI9P3!rmaci|r~CY2fLrD~HK)URgSb?dr5PVnbdD)9h!D6%{Q@j8JyYl|!C-rj~w{>-$Zz zU%!LZ?$0L5L$IVSvYz;T6eThjI6N`@MLp*}E3*};)n@vUwdo;_CG9tcG zvXk}T{Gg;Gb?d~$AtT-X#ZDm$7~zuWF0Z|2$=!J{Ous%HLr+KU`C?rR@UAeg?ZAoivYrAb-|a(2#E1qNpfylrOZ(CKy&`tT z)R^7KB!7D&|B5P$J)#j-$O-v>>bi=inF6FxK!Ct>&~ZYweMp2s1uKtAY%hZrlvpMmyWraxpe&;4%CVghC@pO{~3b)HH3BPzAAq#EBmUR+kb4`~0_wn-+CmaooTLu{)C zZMJW0tUJ7))lJ308@;K8G(Dqmq<=?LQ`EIJQf1rQSNx!Hh=_`9@K=d%_-vGYev;SXN#Zs&De;iKcbo>Z1HTinzZxZ)w`#g zvbVo&4~P>Kxec<$1__MPT^qxlD-ANgGt)z-jO$fkhO7JS8`5iJ$$f+SY`KO73D}qm z;0v-+!%S}yU~mxAElX@LTw->3iHRq zaPA)gzpBPKv}sOqL15^%v_)(P8Vmr39qsAW41B4nn|Q~F zlTShb()K~pQc;Xj1|X{)_0Wrd$lG*>LOR&p;aqu4^G+*GOEni-PpGgN zw%P(=A-dMkJ|MxWQT$=`Nvhl<+@qr*{wIF_hBWR%Zoh+7k{cK#F!<}8P5Q8S2{CSN zB`qcj&UL05s}Y9j<8Vo=h4SENX2ZLm3?FNt#gb`m?HBX(L}D%q?J&)vf>2H9DB7ZP zY(Uk)K<-tH>8k28Oao8m$YpFkK`ol6C1_G*u-03EYEIPX}XTRa> zml#7lJj3DEe*yAAdpYZw!M;oKYhC>}msq-jK2(Tjfe<1bk-TJYjea<{x_6uGUp>O^|swrtJmdQhc F{sVG8`3V33 literal 0 HcmV?d00001 diff --git a/img/dev/en-bloom-update.svg b/img/dev/en-bloom-update.svg new file mode 100644 index 0000000000..b5e88ec93a --- /dev/null +++ b/img/dev/en-bloom-update.svg @@ -0,0 +1,140 @@ + + + + + + +_anonymous_0 + +Automatically Updating Bloom Filters To Track Relevant Transactions +cluster_client + +Client + +cluster_client1 + + +cluster_node + +Full Node + +cluster_node2 + + +cluster_node1 + + + +address + +Address To Match + + +filter1 + + + + + +Filter That +  Matches Address   + + +address->filter1 + + + + + + +filter2 + + + + + +Filter That +  Matches Address   + + +filter1->filter2 + + + + + + +filter3 + + + + + +Filter Updated +    With Outpoint   + + +filter2->filter3 + + + + +match1 +Transaction 1 +Pays Address + + +filter2->match1 + + + + +tx2 + +Transaction 2 + + +tx2->filter3 + + + + +match2 +Transaction 2 +Spends Outpoint + + +filter3->match2 + + + + +match2->mymatch2 + + + + +tx1 + +Transaction 1 + + +tx1->filter2 + + + + +match1->mymatch1 + + + + +match1->filter3 + + + + +