Merge pull request zcash#6933 from zcash/zip-236

Implement "ZIP 236: Blocks must balance exactly" for NU6

.github/workflows/ci.yml

@@ -692,7 +692,7 @@ jobs:
         run: |
           python3 -m venv ./venv
           . ./venv/bin/activate
-          pip install simplejson zmq asyncio
+          pip install zmq asyncio base58
 
   test-rpc:
     name: RPC tests tier ${{ matrix.tier }} platform ${{ matrix.platform }} ${{ matrix.shard }}
@@ -725,7 +725,7 @@ jobs:
         run: |
           python3 -m venv ./venv
           . ./venv/bin/activate
-          pip install simplejson zmq asyncio
+          pip install zmq asyncio base58
 
       - name: Download src artifact
         uses: actions/download-artifact@v4

qa/README.md

@@ -7,20 +7,23 @@ tests locally.
 
 Test dependencies
 =================
+
 Before running the tests, the following must be installed.
 
 Unix
 ----
-The python3-zmq library and simplejson are required. On Ubuntu or Debian they
-can be installed via:
+
+The `zmq` and `base58` Python libraries are required. On Ubuntu or Debian-based
+distributions they can be installed via:
 ```
-sudo apt-get install python3-simplejson python3-zmq
+sudo apt-get install python3-zmq python3-base58
 ```
 
 OS X
 ------
+
 ```
-pip3 install pyzmq simplejson
+pip3 install pyzmq base58
 ```
 
 Running tests

qa/rpc-tests/bip65-cltv-p2p.py

@@ -87,7 +87,6 @@ def get_tests(self):
                                  hashFinalSaplingRoot)
             block.nVersion = 4
             block.vtx.append(spendtx)
-            block.hashMerkleRoot = block.calc_merkle_root()
             block.rehash()
             block.solve()
             self.block_time += 1

qa/rpc-tests/bipdersig-p2p.py

@@ -95,7 +95,6 @@ def get_tests(self):
                                  hashFinalSaplingRoot)
             block.nVersion = 4
             block.vtx.append(spendtx)
-            block.hashMerkleRoot = block.calc_merkle_root()
             block.rehash()
             block.solve()
             self.tip = block.sha256

qa/rpc-tests/coinbase_funding_streams.py

@@ -1,37 +1,59 @@
 #!/usr/bin/env python3
-# Copyright (c) 2020 The Zcash developers
+# Copyright (c) 2020-2024 The Zcash developers
 # Distributed under the MIT software license, see the accompanying
 # file COPYING or https://www.opensource.org/licenses/mit-license.php .
 
+from decimal import Decimal
+
 from test_framework.test_framework import BitcoinTestFramework
+from test_framework.blocktools import create_block, create_coinbase
 from test_framework.mininode import (
-    nuparams,
     fundingstream,
+    nuparams,
+    uint256_from_reversed_hex,
+    CTxOut,
+    COIN,
+)
+from test_framework.script import (
+    CScript,
+    OP_CHECKSIG, OP_DUP, OP_EQUAL, OP_EQUALVERIFY, OP_HASH160,
 )
 from test_framework.util import (
     assert_equal,
     bitcoind_processes,
     connect_nodes,
     start_node,
     BLOSSOM_BRANCH_ID,
-    HEARTWOOD_BRANCH_ID,
     CANOPY_BRANCH_ID,
     NU6_BRANCH_ID,
 )
 
-class CoinbaseFundingStreamsTest (BitcoinTestFramework):
+from base58 import b58decode_check
+
+def redeem_script(addr):
+    raw_addr = b58decode_check(addr)
+    if len(raw_addr) == 22:
+        addr_hash = raw_addr[2:]
+        if raw_addr.startswith(b'\x1C\xBD') or raw_addr.startswith(b'\x1C\xBA'):
+            return CScript([OP_HASH160, addr_hash, OP_EQUAL])
+        elif raw_addr.startswith(b'\x1C\xB8') or raw_addr.startswith(b'\x1D\x25'):
+            return CScript([OP_DUP, OP_HASH160, addr_hash, OP_EQUALVERIFY, OP_CHECKSIG])
+
+    raise ValueError("unrecognized address type")
+
+class CoinbaseFundingStreamsTest(BitcoinTestFramework):
     def __init__(self):
         super().__init__()
         self.num_nodes = 2
         self.cache_behavior = 'clean'
 
     def start_node_with(self, index, extra_args=[]):
         args = [
-            nuparams(BLOSSOM_BRANCH_ID, 1),
-            nuparams(HEARTWOOD_BRANCH_ID, 2),
-            nuparams(CANOPY_BRANCH_ID, 5),
-            nuparams(NU6_BRANCH_ID, 9),
+            nuparams(BLOSSOM_BRANCH_ID, 2),
+            nuparams(CANOPY_BRANCH_ID, 3),
+            nuparams(NU6_BRANCH_ID, 4),
             "-nurejectoldversions=false",
+            "-allowdeprecated=getnewaddress",
             "-allowdeprecated=z_getnewaddress",
             "-allowdeprecated=z_getbalance",
             "-allowdeprecated=z_gettotalbalance",
@@ -43,96 +65,160 @@ def setup_network(self, split=False):
         self.nodes.append(self.start_node_with(0))
         self.nodes.append(self.start_node_with(1))
         connect_nodes(self.nodes[1], 0)
-        self.is_network_split=False
+        self.is_network_split = False
         self.sync_all()
 
-    def run_test (self):
+    def run_test(self):
         # Generate a shielded address for node 1 for miner rewards,
         miner_addr = self.nodes[1].z_getnewaddress('sapling')
 
-        # Generate a shielded address (belonging to node 0) for funding stream
-        # rewards.
-        fs_addr = self.nodes[0].z_getnewaddress('sapling')
+        # Generate Sapling and transparent addresses (belonging to node 0) for funding stream rewards.
+        # For transparent funding stream outputs, strictly speaking the protocol spec only specifies
+        # paying to a P2SH multisig, but P2PKH funding addresses also work in zcashd.
+        fs_sapling = self.nodes[0].z_getnewaddress('sapling')
+        fs_transparent = self.nodes[0].getnewaddress()
 
-        # Generate past heartwood activation we won't need node 1 from this
-        # point onward except to check miner reward balances
+        print("Activating Blossom")
         self.nodes[1].generate(2)
         self.sync_all()
+        # We won't need to mine on node 1 from this point onward.
+
+        assert_equal(self.nodes[1].getblockchaininfo()['blocks'], 2)
 
-        # Restart node 0 with funding streams.
+        # Restart both nodes with funding streams.
         self.nodes[0].stop()
         bitcoind_processes[0].wait()
-        self.nodes[0] = self.start_node_with(0, [
+        self.nodes[1].stop()
+        bitcoind_processes[1].wait()
+        new_args = [
             "-mineraddress=%s" % miner_addr,
             "-minetolocalwallet=0",
-            fundingstream(0, 5, 9, [fs_addr]),
-            fundingstream(1, 5, 9, [fs_addr]),
-            fundingstream(2, 5, 9, [fs_addr]),
-            fundingstream(3, 9, 13, [fs_addr, fs_addr]),
-            fundingstream(4, 9, 13, ["DEFERRED_POOL", "DEFERRED_POOL"]),
-        ])
+            fundingstream(0, 3, 4, [fs_sapling]), # FS_ZIP214_BP: 7%
+            fundingstream(1, 3, 4, [fs_sapling]), # FS_ZIP214_ZF: 5%
+            fundingstream(2, 3, 4, [fs_sapling]), # FS_ZIP214_MG: 8%
+            fundingstream(3, 4, 8, [fs_transparent, fs_transparent]),   # FS_ZIPTBD_ZCG: 8%
+            fundingstream(4, 4, 8, ["DEFERRED_POOL", "DEFERRED_POOL"]), # FS_ZIPTBD_LOCKBOX: 12%
+        ]
+        self.nodes[0] = self.start_node_with(0, new_args)
+        self.nodes[1] = self.start_node_with(1, new_args)
         connect_nodes(self.nodes[1], 0)
         self.sync_all()
 
-        print("Generate to just prior to Canopy activation")
-        self.nodes[0].generate(2)
+        print("Activating Heartwood & Canopy")
+        self.nodes[0].generate(1)
         self.sync_all()
 
-        assert_equal(self.nodes[1].getblockchaininfo()['blocks'], 4);
+        assert_equal(self.nodes[1].getblockchaininfo()['blocks'], 3)
 
         # All miner addresses belong to node 1; check balances
         # Miner rewards are uniformly 5 zec per block (all test heights are below the first halving)
         walletinfo = self.nodes[1].getwalletinfo()
-        assert_equal(walletinfo['immature_balance'], 10)
-        assert_equal(walletinfo['balance'], 0)
-        assert_equal(self.nodes[1].z_getbalance(miner_addr, 0), 10)
-        assert_equal(self.nodes[1].z_getbalance(miner_addr), 10)
-
-        print("Activating Canopy")
-        self.nodes[0].generate(4)
+        assert_equal(Decimal(walletinfo['immature_balance']), Decimal('15'))
+        assert_equal(Decimal(walletinfo['balance']), Decimal('0'))
+        assert_equal(Decimal(self.nodes[1].z_getbalance(miner_addr, 0)), Decimal('5'))
+        assert_equal(Decimal(self.nodes[1].z_getbalance(miner_addr)), Decimal('5'))
+
+        # Check that the node 0 private balance has been augmented by the funding stream payment.
+        assert_equal(Decimal(self.nodes[0].z_getbalance(fs_sapling, 0)), Decimal('1.25'))
+        assert_equal(Decimal(self.nodes[0].z_getbalance(fs_sapling)), Decimal('1.25'))
+        assert_equal(Decimal(self.nodes[0].z_gettotalbalance()['private']), Decimal('1.25'))
+        assert_equal(Decimal(self.nodes[0].z_gettotalbalance()['total']), Decimal('1.25'))
+
+        print("Activating NU5 & NU6")
+        self.nodes[0].generate(1)
         self.sync_all()
 
-        assert_equal(self.nodes[1].getblockchaininfo()['blocks'], 8);
+        assert_equal(self.nodes[1].getblockchaininfo()['blocks'], 4)
 
         # check that miner payments made it to node 1's wallet
         walletinfo = self.nodes[1].getwalletinfo()
-        assert_equal(walletinfo['immature_balance'], 10)
-        assert_equal(walletinfo['balance'], 0)
-        assert_equal(self.nodes[1].z_getbalance(miner_addr, 0), 30)
-        assert_equal(self.nodes[1].z_getbalance(miner_addr), 30)
-
-        # Check that the node 0 private balance has been augmented by the
-        # funding stream payments.
-        # Prior to NU6, the total per-block value of development funding is 1.25 ZEC
-        assert_equal(self.nodes[0].z_getbalance(fs_addr, 0), 5)
-        assert_equal(self.nodes[0].z_getbalance(fs_addr), 5)
-        assert_equal(self.nodes[0].z_gettotalbalance()['private'], '5.00')
-        assert_equal(self.nodes[0].z_gettotalbalance()['total'], '5.00')
-
-        print("Activating NU6")
+        assert_equal(Decimal(walletinfo['immature_balance']), Decimal('15'))
+        assert_equal(Decimal(walletinfo['balance']), Decimal('0'))
+        assert_equal(Decimal(self.nodes[1].z_getbalance(miner_addr, 0)), Decimal('10'))
+        assert_equal(Decimal(self.nodes[1].z_getbalance(miner_addr)), Decimal('10'))
+
+        # Check that the node 0 *immature* transparent balance has been augmented by the funding
+        # stream payment:
+        # * 0.5 ZEC per block to fs_transparent
+        # * 0.75 ZEC per block to the lockbox
+        walletinfo = self.nodes[0].getwalletinfo()
+        assert_equal(Decimal(walletinfo['immature_balance']), Decimal('0.5'))
+        assert_equal(Decimal(walletinfo['shielded_balance']), Decimal('1.25'))
+
+        # Check that the node 0 lockbox balance has been augmented by the funding stream payment.
+        lastBlock = self.nodes[0].getblock('4')
+        def check_lockbox(blk, expected):
+            lockbox = next(elem for elem in blk['valuePools'] if elem['id'] == "lockbox")
+            assert_equal(Decimal(lockbox['chainValue']), expected)
+        check_lockbox(lastBlock, Decimal('0.75'))
+
+        # Invalidate the last block so that we can steal its `hashBlockCommitments` field.
+        self.nodes[0].invalidateblock(lastBlock['hash'])
+        self.nodes[1].invalidateblock(lastBlock['hash'])
+        self.sync_all()
+
+        assert_equal(self.nodes[1].getblockchaininfo()['blocks'], 3)
+
+        walletinfo = self.nodes[0].getwalletinfo()
+        assert_equal(Decimal(walletinfo['immature_balance']), Decimal('0'))
+        assert_equal(Decimal(walletinfo['shielded_balance']), Decimal('1.25'))
+
+        coinbaseTx = create_coinbase(
+            4,
+            after_blossom=True,
+            outputs=[CTxOut(int(0.5 * COIN), redeem_script(fs_transparent))],
+            lockboxvalue=int(0.75 * COIN),
+        )
+        block = create_block(
+            uint256_from_reversed_hex(lastBlock['previousblockhash']),
+            coinbaseTx,
+            lastBlock['time'],
+            hashBlockCommitments=uint256_from_reversed_hex(lastBlock['blockcommitments']),
+        )
+
+        # A coinbase transaction that claims less than the maximum amount should be invalid after NU6.
+        block.vtx[0].vout[0].nValue -= 1  # too little
+        block.vtx[0].rehash()
+        block.rehash()
+        block.solve()
+        assert_equal(self.nodes[0].submitblock(block.serialize().hex()), 'bad-cb-not-exact')
+
+        # A coinbase transaction that claims more than the maximum amount should also be invalid
+        # (with a different error).
+        block.vtx[0].vout[0].nValue += 2  # too much
+        block.vtx[0].rehash()
+        block.rehash()
+        block.solve()
+        assert_equal(self.nodes[0].submitblock(block.serialize().hex()), 'bad-cb-amount')
+
+        # A coinbase transaction that claims exactly the maximum amount should be valid.
+        block.vtx[0].vout[0].nValue -= 1  # just right
+        block.vtx[0].rehash()
+        block.rehash()
+        block.solve()
+        assert_equal(self.nodes[0].submitblock(block.serialize().hex()), None)
+        self.sync_all()
+
+        assert_equal(self.nodes[1].getblockchaininfo()['blocks'], 4)
+
+        walletinfo = self.nodes[0].getwalletinfo()
+        assert_equal(Decimal(walletinfo['immature_balance']), Decimal('0.5'))
+        assert_equal(Decimal(walletinfo['shielded_balance']), Decimal('1.25'))
+
+        check_lockbox(self.nodes[0].getblock('4'), Decimal('0.75'))
+
+        print("Mining past the end of the funding streams")
         self.nodes[0].generate(4)
         self.sync_all()
 
-        walletinfo = self.nodes[1].getwalletinfo()
-        assert_equal(walletinfo['immature_balance'], 10)
-        assert_equal(walletinfo['balance'], 0)
-        assert_equal(self.nodes[1].z_getbalance(miner_addr, 0), 50)
-        assert_equal(self.nodes[1].z_getbalance(miner_addr), 50)
-
-        # check that the node 0 private balance has been augmented by the
-        # funding stream payments:
-        # * 0.5 ZEC per block to fs_addr
-        # * 0.75 ZEC per block to the lockbox
-        assert_equal(self.nodes[0].z_getbalance(fs_addr, 0), 7)
-        assert_equal(self.nodes[0].z_getbalance(fs_addr), 7)
-        assert_equal(self.nodes[0].z_gettotalbalance()['private'], '7.00')
-        assert_equal(self.nodes[0].z_gettotalbalance()['total'], '7.00')
-
-        # check that the node 0 lockbox balance has been augmented by the
-        # funding stream payments.
-        valuePools = self.nodes[0].getblock("12")['valuePools']
-        lockbox = next(elem for elem in valuePools if elem['id'] == "lockbox")
-        assert_equal(lockbox['chainValue'], 3)
+        assert_equal(self.nodes[1].getblockchaininfo()['blocks'], 8)
+
+        walletinfo = self.nodes[0].getwalletinfo()
+        assert_equal(Decimal(walletinfo['immature_balance']), 4 * Decimal('0.5'))
+        assert_equal(Decimal(walletinfo['shielded_balance']), Decimal('1.25'))
+
+        check_lockbox(self.nodes[0].getblock('7'), 4 * Decimal('0.75'))
+        check_lockbox(self.nodes[0].getblock('8'), 4 * Decimal('0.75'))
 
 if __name__ == '__main__':
     CoinbaseFundingStreamsTest().main()

qa/rpc-tests/feature_zip239.py

@@ -13,7 +13,7 @@
     msg_getdata,
     msg_mempool,
     msg_reject,
-    uint256_from_str,
+    uint256_from_reversed_hex,
 )
 from test_framework.test_framework import BitcoinTestFramework
 from test_framework.util import (
@@ -26,7 +26,6 @@
     assert_false,
     assert_true,
     fail,
-    hex_str_to_bytes,
     nuparams,
     p2p_port,
     start_nodes,
@@ -202,16 +201,16 @@ def run_test(self):
             'address': node1_taddr,
             'amount': 1,
         }], 1, LEGACY_DEFAULT_FEE, 'AllowRevealedRecipients')
-        v4_txid = uint256_from_str(hex_str_to_bytes(
+        v4_txid = uint256_from_reversed_hex(
             wait_and_assert_operationid_status(self.nodes[0], opid)
-        )[::-1])
+        )
 
         # Add v5 transaction to the mempool.
         v5_txid = self.nodes[0].sendtoaddress(node1_taddr, 1, "", "", True)
         v5_tx = self.nodes[0].getrawtransaction(v5_txid, 1)
         assert_equal(v5_tx['version'], 5)
-        v5_txid = uint256_from_str(hex_str_to_bytes(v5_txid)[::-1])
-        v5_auth_digest = uint256_from_str(hex_str_to_bytes(v5_tx['authdigest'])[::-1])
+        v5_txid = uint256_from_reversed_hex(v5_txid)
+        v5_auth_digest = uint256_from_reversed_hex(v5_tx['authdigest'])
 
         # Wait for the mempools to sync.
         self.sync_all()