Backport missing PRs from Bitcoin 0.14 - Part 7

.gitignore

@@ -108,7 +108,6 @@ coverage_percent.txt
 linux-coverage-build
 linux-build
 win32-build
-qa/pull-tester/run-bitcoind-for-test.sh
 qa/pull-tester/tests_config.py
 qa/cache/*
 

configure.ac

@@ -513,8 +513,6 @@ if test x$TARGET_OS = xdarwin; then
 fi
 
 AC_CHECK_HEADERS([endian.h sys/endian.h byteswap.h stdio.h stdlib.h unistd.h strings.h sys/types.h sys/stat.h sys/select.h sys/prctl.h])
-AC_SEARCH_LIBS([getaddrinfo_a], [anl], [AC_DEFINE(HAVE_GETADDRINFO_A, 1, [Define this symbol if you have getaddrinfo_a])])
-AC_SEARCH_LIBS([inet_pton], [nsl resolv], [AC_DEFINE(HAVE_INET_PTON, 1, [Define this symbol if you have inet_pton])])
 
 AC_CHECK_DECLS([strnlen])
 
@@ -1080,7 +1078,6 @@ AC_SUBST(ZMQ_LIBS)
 AC_SUBST(PROTOBUF_LIBS)
 AC_SUBST(QR_LIBS)
 AC_CONFIG_FILES([Makefile src/Makefile doc/man/Makefile share/setup.nsi share/qt/Info.plist src/test/buildenv.py])
-AC_CONFIG_FILES([qa/pull-tester/run-bitcoind-for-test.sh],[chmod +x qa/pull-tester/run-bitcoind-for-test.sh])
 AC_CONFIG_FILES([qa/pull-tester/tests_config.py],[chmod +x qa/pull-tester/tests_config.py])
 AC_CONFIG_FILES([contrib/devtools/split-debug.sh],[chmod +x contrib/devtools/split-debug.sh])
 AC_CONFIG_LINKS([qa/pull-tester/rpc-tests.py:qa/pull-tester/rpc-tests.py])

doc/build-windows.md

@@ -7,16 +7,18 @@ Most developers use cross-compilation from Ubuntu to build executables for
 Windows. This is also used to build the release binaries.
 
 While there are potentially a number of ways to build on Windows (for example using msys / mingw-w64),
-using the Windows Subsystem For Linux is the most straight forward.  If you are building with
-an alternative method, please contribute the instructions here for others who are running versions
+using the Windows Subsystem For Linux is the most straightforward. If you are building with
+another method, please contribute the instructions here for others who are running versions
 of Windows that are not compatible with the Windows Subsystem for Linux.
 
-Compiling with the Windows Subsystem For Linux
--------------------
+Compiling with Windows Subsystem For Linux
+-------------------------------------------
 
-With Windows 10, Microsoft has released a new feature named the
-[Windows Subsystem for Linux](https://msdn.microsoft.com/commandline/wsl/about).  This feature allows you to run a bash shell directly on Windows in an Ubuntu based
-environment.  Within this environment you can cross compile for Windows without the need for a separate Linux VM or Server.
+With Windows 10, Microsoft has released a new feature named the [Windows
+Subsystem for Linux](https://msdn.microsoft.com/commandline/wsl/about). This
+feature allows you to run a bash shell directly on Windows in an Ubuntu based
+environment. Within this environment you can cross compile for Windows without
+the need for a separate Linux VM or Server.
 
 This feature is not supported in versions of Windows prior to Windows 10 or on Windows Server SKUs.
 
@@ -37,9 +39,6 @@ To get the bash shell, you must first activate the feature in Windows.
   * Create a new UNIX user account (this is a separate account from your Windows account)
 
 After the bash shell is active, you can follow the instructions below for Windows 64-bit Cross-compilation.
-When building dependencies within the 'depends' folder, you may encounter an error building
-the protobuf dependency.  If this occurs, re-run the command with sudo.  This is likely
-a bug with the Windows Subsystem for Linux feature and may be fixed with a future update.
 
 Cross-compilation
 -------------------
@@ -48,28 +47,55 @@ These steps can be performed on, for example, an Ubuntu VM. The depends system
 will also work on other Linux distributions, however the commands for
 installing the toolchain will be different.
 
-Make sure you install the build requirements mentioned in
-[build-unix.md](/doc/build-unix.md).
-Then, install the toolchains and curl:
+First, install the general dependencies:
+
+    sudo apt-get install build-essential libtool autotools-dev automake pkg-config bsdmainutils curl
+
+A host toolchain (`build-essential`) is necessary because some dependency
+packages (such as `protobuf`) need to build host utilities that are used in the
+build process.
+
+## Building for 64-bit Windows
+
+To build executables for Windows 64-bit, install the following dependencies:
 
-    sudo apt-get install g++-mingw-w64-i686 mingw-w64-i686-dev g++-mingw-w64-x86-64 mingw-w64-x86-64-dev curl
+    sudo apt-get install g++-mingw-w64-x86-64 mingw-w64-x86-64-dev
 
-To build executables for Windows 32-bit:
+Then build using:
 
     cd depends
-    make HOST=i686-w64-mingw32 -j4
+    make HOST=x86_64-w64-mingw32 -j4
     cd ..
     ./autogen.sh # not required when building from tarball
-    ./configure --prefix=`pwd`/depends/i686-w64-mingw32
+    CONFIG_SITE=$PWD/depends/x86_64-w64-mingw32/share/config.site ./configure --prefix=/
     make
 
-To build executables for Windows 64-bit:
+## Building for 32-bit Windows
+
+To build executables for Windows 32-bit, install the following dependencies:
+
+    sudo apt-get install g++-mingw-w64-i686 mingw-w64-i686-dev 
+
+Then build using:
 
     cd depends
-    make HOST=x86_64-w64-mingw32 -j4
+    make HOST=i686-w64-mingw32 -j4
     cd ..
     ./autogen.sh # not required when building from tarball
-    ./configure --prefix=`pwd`/depends/x86_64-w64-mingw32
+    CONFIG_SITE=$PWD/depends/i686-w64-mingw32/share/config.site ./configure --prefix=/
     make
 
+## Depends system
+
 For further documentation on the depends system see [README.md](../depends/README.md) in the depends directory.
+
+Installation
+-------------
+
+After building using the Windows subsystem it can be useful to copy the compiled
+executables to a directory on the windows drive in the same directory structure
+as they appear in the release `.zip` archive. This can be done in the following
+way. This will install to `c:\workspace\dash`, for example:
+
+    make install DESTDIR=/mnt/c/workspace/dash
+

doc/developer-notes.md

@@ -410,6 +410,37 @@ GUI
     should not interact with the user. That's where View classes come in. The converse also
     holds: try to not directly access core data structures from Views.
 
+Subtrees
+----------
+
+Several parts of the repository are subtrees of software maintained elsewhere.
+
+Some of these are maintained by active developers of Bitcoin Core, in which case changes should probably go
+directly upstream without being PRed directly against the project.  They will be merged back in the next
+subtree merge.
+
+Others are external projects without a tight relationship with our project.  Changes to these should also
+be sent upstream but bugfixes may also be prudent to PR against Dash Core so that they can be integrated
+quickly.  Cosmetic changes should be purely taken upstream.
+
+There is a tool in contrib/devtools/git-subtree-check.sh to check a subtree directory for consistency with
+its upstream repository.
+
+Current subtrees include:
+
+- src/leveldb
+  - Upstream at https://github.com/google/leveldb ; Maintained by Google, but open important PRs to Core to avoid delay
+
+- src/libsecp256k1
+  - Upstream at https://github.com/bitcoin-core/secp256k1/ ; actively maintaned by Core contributors.
+
+- src/crypto/ctaes
+  - Upstream at https://github.com/bitcoin-core/ctaes ; actively maintained by Core contributors.
+
+- src/univalue
+  - Upstream at https://github.com/jgarzik/univalue ; report important PRs to Core to avoid delay.
+
+
 Git and github tips
 ---------------------
 

qa/pull-tester/rpc-tests.py

@@ -109,78 +109,87 @@
 
 testScripts = [
     # longest test should go first, to favor running tests in parallel
-    'p2p-fullblocktest.py', # NOTE: needs dash_hash to pass
+    'wallet-hd.py',
     'walletbackup.py',
-    'bip68-112-113-p2p.py',
+    # vv Tests less than 5m vv
+    'p2p-fullblocktest.py', # NOTE: needs dash_hash to pass
+    'fundrawtransaction.py',
+    'fundrawtransaction-hd.py',
+    # vv Tests less than 2m vv
     'wallet.py',
     'wallet-accounts.py',
-    'wallet-hd.py',
     'listtransactions.py',
+    # vv Tests less than 60s vv
+    'sendheaders.py', # NOTE: needs dash_hash to pass
+    'zapwallettxes.py',
+    'importmulti.py',
+    'mempool_limit.py',
+    'merkle_blocks.py',
     'receivedby.py',
+    'abandonconflict.py',
+    'bip68-112-113-p2p.py',
+    'rawtransactions.py',
+    'reindex.py',
+    # vv Tests less than 30s vv
     'mempool_resurrect_test.py',
     'txn_doublespend.py --mineblock',
     'txn_clone.py',
     'getchaintips.py',
-    'rawtransactions.py',
     'rest.py',
     'mempool_spendcoinbase.py',
     'mempool_reorg.py',
-    'mempool_limit.py',
     'httpbasics.py',
     'multi_rpc.py',
-    'zapwallettxes.py',
     'proxy_test.py',
-    'merkle_blocks.py',
-    'fundrawtransaction.py',
-    'fundrawtransaction-hd.py',
     'signrawtransactions.py',
     'nodehandling.py',
-    'reindex.py',
     'addressindex.py',
     'timestampindex.py',
     'spentindex.py',
     'decodescript.py',
     'blockchain.py',
     'disablewallet.py',
-    'sendheaders.py', # NOTE: needs dash_hash to pass
     'keypool.py',
     'keypool-hd.py',
     'p2p-mempool.py',
     'prioritise_transaction.py',
     'invalidblockrequest.py', # NOTE: needs dash_hash to pass
     'invalidtxrequest.py', # NOTE: needs dash_hash to pass
-    'abandonconflict.py',
     'p2p-versionbits-warning.py',
     'preciousblock.py',
     'importprunedfunds.py',
     'signmessages.py',
-    'importmulti.py',
 ]
 if ENABLE_ZMQ:
     testScripts.append('zmq_test.py')
 
 testScriptsExt = [
+    # 'pruning.py', # Prune mode is incompatible with -txindex.
+    # vv Tests less than 20m vv
+    'smartfees.py',
+    # vv Tests less than 5m vv
+    'maxuploadtarget.py',
+    'mempool_packages.py',
+    # vv Tests less than 2m vv
+    'bip68-sequence.py',
+    'getblocktemplate_longpoll.py',  # FIXME: "socket.error: [Errno 54] Connection reset by peer" on my Mac, same as  https://github.com/bitcoin/bitcoin/issues/6651
+    # vv Tests less than 60s vv
     'bip9-softforks.py',
+    'p2p-feefilter.py',
+    'rpcbind_test.py',
+    # vv Tests less than 30s vv
     'bip65-cltv.py',
     'bip65-cltv-p2p.py', # NOTE: needs dash_hash to pass
-    'bip68-sequence.py',
     'bipdersig-p2p.py', # NOTE: needs dash_hash to pass
     'bipdersig.py',
-    'getblocktemplate_longpoll.py', # FIXME: "socket.error: [Errno 54] Connection reset by peer" on my Mac, same as  https://github.com/bitcoin/bitcoin/issues/6651
     'getblocktemplate_proposals.py',
     'txn_doublespend.py',
     'txn_clone.py --mineblock',
     'forknotify.py',
     'invalidateblock.py',
-    'rpcbind_test.py',
-    'smartfees.py',
     'maxblocksinflight.py',
     'p2p-acceptblock.py', # NOTE: needs dash_hash to pass
-    'mempool_packages.py',
-    'maxuploadtarget.py',
     # 'replace-by-fee.py', # RBF is disabled in Dash Core
-    'p2p-feefilter.py',
-    # 'pruning.py', # leave pruning last as it takes a REALLY long time #### Prune mode is incompatible with -txindex.
 ]
 
 

qa/rpc-tests/fundrawtransaction.py

@@ -515,7 +515,7 @@ def run_test(self):
         fundedTx = self.nodes[1].fundrawtransaction(rawTx)
 
         #now we need to unlock
-        self.nodes[1].walletpassphrase("test", 100)
+        self.nodes[1].walletpassphrase("test", 600)
         signedTx = self.nodes[1].signrawtransaction(fundedTx['hex'])
         txId = self.nodes[1].sendrawtransaction(signedTx['hex'])
         self.nodes[1].generate(1)

qa/rpc-tests/maxuploadtarget.py

@@ -81,49 +81,19 @@ def received_pong():
 
 class MaxUploadTest(BitcoinTestFramework):
 
-    def add_options(self, parser):
-        parser.add_option("--testbinary", dest="testbinary",
-                          default=os.getenv("DASHD", "dashd"),
-                          help="dashd binary to test")
-
     def __init__(self):
         super().__init__()
         self.setup_clean_chain = True
         self.num_nodes = 1
 
-        self.utxo = []
-        self.txouts = gen_return_txouts()
+        # Cache for utxos, as the listunspent may take a long time later in the test
+        self.utxo_cache = []
 
     def setup_network(self):
         # Start a node with maxuploadtarget of 200 MB (/24h)
         self.nodes = []
         self.nodes.append(start_node(0, self.options.tmpdir, ["-debug", "-maxuploadtarget=200", "-blockmaxsize=999000"]))
 
-    def mine_full_block(self, node, address):
-        # Want to create a full block
-        # We'll generate a 66k transaction below, and 14 of them is close to the 1MB block limit
-        for j in range(14):
-            if len(self.utxo) < 14:
-                self.utxo = node.listunspent()
-            inputs=[]
-            outputs = {}
-            t = self.utxo.pop()
-            inputs.append({ "txid" : t["txid"], "vout" : t["vout"]})
-            remchange = t["amount"] - Decimal("0.001000")
-            outputs[address]=remchange
-            # Create a basic transaction that will send change back to ourself after account for a fee
-            # And then insert the 128 generated transaction outs in the middle rawtx[92] is where the #
-            # of txouts is stored and is the only thing we overwrite from the original transaction
-            rawtx = node.createrawtransaction(inputs, outputs)
-            newtx = rawtx[0:92]
-            newtx = newtx + self.txouts
-            newtx = newtx + rawtx[94:]
-            # Appears to be ever so slightly faster to sign with SIGHASH_NONE
-            signresult = node.signrawtransaction(newtx,None,None,"NONE")
-            txid = node.sendrawtransaction(signresult["hex"], True)
-        # Mine a full sized block which will be these transactions we just created
-        node.generate(1)
-
     def run_test(self):
         # Before we connect anything, we first set the time on the node
         # to be in the past, otherwise things break because the CNode
@@ -151,7 +121,7 @@ def run_test(self):
         # Test logic begins here
 
         # Now mine a big block
-        self.mine_full_block(self.nodes[0], self.nodes[0].getnewaddress())
+        mine_large_block(self.nodes[0], self.utxo_cache)
 
         # Store the hash; we'll request this later
         big_old_block = self.nodes[0].getbestblockhash()
@@ -162,11 +132,10 @@ def run_test(self):
         self.nodes[0].setmocktime(int(time.time()) - 2*60*60*24)
 
         # Mine one more block, so that the prior block looks old
-        self.mine_full_block(self.nodes[0], self.nodes[0].getnewaddress())
+        mine_large_block(self.nodes[0], self.utxo_cache)
 
         # We'll be requesting this new block too
         big_new_block = self.nodes[0].getbestblockhash()
-        new_block_size = self.nodes[0].getblock(big_new_block)['size']
         big_new_block = int(big_new_block, 16)
 
         # test_nodes[0] will test what happens if we just keep requesting the