backport bitcoin#16509 and add devnet test

test/functional/combine_logs.py

@@ -6,6 +6,7 @@
 
 import argparse
 from collections import defaultdict, namedtuple
+import glob
 import heapq
 import itertools
 import os
@@ -48,9 +49,17 @@ def read_logs(tmp_dir):
     Delegates to generator function get_log_events() to provide individual log events
     for each of the input log files."""
 
+    # Find out what the folder is called that holds the debug.log file
+    chain = glob.glob("{}/node0/*/debug.log".format(tmp_dir))
+    if chain:
+        chain = chain[0]  # pick the first one if more than one chain was found (should never happen)
+        chain = re.search('node0/(.+?)/debug\.log$', chain).group(1)  # extract the chain name
+    else:
+        chain = 'regtest'  # fallback to regtest (should only happen when none exists)
+
     files = [("test", "%s/test_framework.log" % tmp_dir)]
     for i in itertools.count():
-        logfile = "{}/node{}/regtest/debug.log".format(tmp_dir, i)
+        logfile = "{}/node{}/{}/debug.log".format(tmp_dir, i, chain)
         if not os.path.isfile(logfile):
             break
         files.append(("node%d" % i, logfile))

test/functional/feature_assumevalid.py

@@ -73,12 +73,8 @@ def send_blocks_until_disconnected(self, p2p_conn):
                 break
             try:
                 p2p_conn.send_message(msg_block(self.blocks[i]))
-            # TODO There is a race condition between send_message and on_close which causes an AttributError on Travis
-            # We can reenable the correct exception handling and the assert when Bitcoin 0.16 mininode.py changes have been
-            # backported
-            #except IOError as e:
-            except:
-                #assert not p2p_conn.is_connected
+            except IOError:
+                assert not p2p_conn.is_connected
                 break
 
     def assert_blockchain_height(self, node, height):

test/functional/feature_blocksdir.py

@@ -19,7 +19,7 @@ def set_test_params(self):
     def run_test(self):
         self.stop_node(0)
         shutil.rmtree(self.nodes[0].datadir)
-        initialize_datadir(self.options.tmpdir, 0)
+        initialize_datadir(self.options.tmpdir, 0, self.chain)
         self.log.info("Starting with non exiting blocksdir ...")
         blocksdir_path = os.path.join(self.options.tmpdir, 'blocksdir')
         self.nodes[0].assert_start_raises_init_error(["-blocksdir=" + blocksdir_path], 'Error: Specified blocks directory "' +
@@ -29,8 +29,8 @@ def run_test(self):
         self.start_node(0, ["-blocksdir=" + blocksdir_path])
         self.log.info("mining blocks..")
         self.nodes[0].generate(10)
-        assert os.path.isfile(os.path.join(blocksdir_path, "regtest", "blocks", "blk00000.dat"))
-        assert os.path.isdir(os.path.join(self.nodes[0].datadir, "regtest", "blocks", "index"))
+        assert os.path.isfile(os.path.join(blocksdir_path, self.chain, "blocks", "blk00000.dat"))
+        assert os.path.isdir(os.path.join(self.nodes[0].datadir, self.chain, "blocks", "index"))
 
 
 if __name__ == '__main__':

test/functional/feature_config_args.py

@@ -31,7 +31,7 @@ def run_test(self):
         # Check that using non-existent datadir in conf file fails
         conf_file = os.path.join(default_data_dir, "dash.conf")
 
-        # datadir needs to be set before [regtest] section
+        # datadir needs to be set before [chain] section
         conf_file_contents = open(conf_file, encoding='utf8').read()
         with open(conf_file, 'w', encoding='utf8') as f:
             f.write("datadir=" + new_data_dir + "\n")
@@ -43,13 +43,13 @@ def run_test(self):
         os.mkdir(new_data_dir)
         self.start_node(0, ['-conf='+conf_file, '-wallet=w1'])
         self.stop_node(0)
-        assert os.path.exists(os.path.join(new_data_dir, 'regtest', 'wallets', 'w1'))
+        assert os.path.exists(os.path.join(new_data_dir, self.chain, 'wallets', 'w1'))
 
         # Ensure command line argument overrides datadir in conf
         os.mkdir(new_data_dir_2)
         self.nodes[0].datadir = new_data_dir_2
         self.start_node(0, ['-datadir='+new_data_dir_2, '-conf='+conf_file, '-wallet=w2'])
-        assert os.path.exists(os.path.join(new_data_dir_2, 'regtest', 'wallets', 'w2'))
+        assert os.path.exists(os.path.join(new_data_dir_2, self.chain, 'wallets', 'w2'))
 
 if __name__ == '__main__':
     ConfArgsTest().main()

test/functional/feature_logging.py

@@ -15,19 +15,19 @@ def set_test_params(self):
 
     def run_test(self):
         # test default log file name
-        assert os.path.isfile(os.path.join(self.nodes[0].datadir, "regtest", "debug.log"))
+        assert os.path.isfile(os.path.join(self.nodes[0].datadir, self.chain, "debug.log"))
 
         # test alternative log file name in datadir
         self.restart_node(0, ["-debuglogfile=foo.log"])
-        assert os.path.isfile(os.path.join(self.nodes[0].datadir, "regtest", "foo.log"))
+        assert os.path.isfile(os.path.join(self.nodes[0].datadir, self.chain, "foo.log"))
 
         # test alternative log file name outside datadir
         tempname = os.path.join(self.options.tmpdir, "foo.log")
         self.restart_node(0, ["-debuglogfile=%s" % tempname])
         assert os.path.isfile(tempname)
 
         # check that invalid log (relative) will cause error
-        invdir = os.path.join(self.nodes[0].datadir, "regtest", "foo")
+        invdir = os.path.join(self.nodes[0].datadir, self.chain, "foo")
         invalidname = os.path.join("foo", "foo.log")
         self.stop_node(0)
         exp_stderr = "Error: Could not open debug log file \S+$"

test/functional/feature_pruning.py

@@ -45,7 +45,7 @@ def set_test_params(self):
     def setup_network(self):
         self.setup_nodes()
 
-        self.prunedir = os.path.join(self.nodes[2].datadir, 'regtest', 'blocks', '')
+        self.prunedir = os.path.join(self.nodes[2].datadir, self.chain, 'blocks', '')
 
         connect_nodes(self.nodes[0], 1)
         connect_nodes(self.nodes[1], 2)
@@ -256,7 +256,7 @@ def prune(index, expected_ret=None):
                 assert_equal(ret, expected_ret)
 
         def has_block(index):
-            return os.path.isfile(os.path.join(self.nodes[node_number].datadir, "regtest", "blocks", "blk{:05}.dat".format(index)))
+            return os.path.isfile(os.path.join(self.nodes[node_number].datadir, self.chain, "blocks", "blk{:05}.dat".format(index)))
 
         # should not prune because chain tip of node 3 (995) < PruneAfterHeight (1000)
         assert_raises_rpc_error(-1, "Blockchain is too short for pruning", node.pruneblockchain, height(500))

test/functional/interface_bitcoin_cli.py

@@ -25,7 +25,7 @@ def run_test(self):
         rpc_response = self.nodes[0].getblockchaininfo()
         assert_equal(cli_response, rpc_response)
 
-        user, password = get_auth_cookie(self.nodes[0].datadir)
+        user, password = get_auth_cookie(self.nodes[0].datadir, self.chain)
 
         self.log.info("Test -stdinrpcpass option")
         assert_equal(0, self.nodes[0].cli('-rpcuser=%s' % user, '-stdinrpcpass', input=password).getblockcount())

test/functional/mempool_persist.py

@@ -93,8 +93,8 @@ def run_test(self):
         self.start_node(0)
         wait_until(lambda: len(self.nodes[0].getrawmempool()) == 5)
 
-        mempooldat0 = os.path.join(self.nodes[0].datadir, 'regtest', 'mempool.dat')
-        mempooldat1 = os.path.join(self.nodes[1].datadir, 'regtest', 'mempool.dat')
+        mempooldat0 = os.path.join(self.nodes[0].datadir, self.chain, 'mempool.dat')
+        mempooldat1 = os.path.join(self.nodes[1].datadir, self.chain, 'mempool.dat')
         self.log.debug("Remove the mempool.dat file. Verify that savemempool to disk via RPC re-creates it")
         os.remove(mempooldat0)
         self.nodes[0].savemempool()

test/functional/mining_basic.py

@@ -37,7 +37,7 @@ def run_test(self):
         self.log.info('getmininginfo')
         mining_info = node.getmininginfo()
         assert_equal(mining_info['blocks'], 200)
-        assert_equal(mining_info['chain'], 'regtest')
+        assert_equal(mining_info['chain'], self.chain)
         assert_equal(mining_info['currentblocksize'], 0)
         assert_equal(mining_info['currentblocktx'], 0)
         assert_equal(mining_info['difficulty'], Decimal('4.656542373906925E-10'))

test/functional/p2p_connect_to_devnet.py

@@ -0,0 +1,31 @@
+#!/usr/bin/env python3
+# Copyright (c) 2021 The Dash Core developers
+# Distributed under the MIT software license, see the accompanying
+# file COPYING or http://www.opensource.org/licenses/mit-license.php.
+"""Test being able to connect to the same devnet"""
+
+from test_framework.test_framework import BitcoinTestFramework
+from test_framework.util import connect_nodes_bi
+
+class ConnectDevnetNodes(BitcoinTestFramework):
+    def set_test_params(self):
+        self.setup_clean_chain = True
+        self.chain = "devnet"
+        self.num_nodes = 2
+
+    def setup_network(self):
+        self.add_nodes(self.num_nodes)
+        self.start_node(0)
+        self.start_node(1)
+        connect_nodes_bi(self.nodes, 0, 1)
+        self.sync_all()
+
+
+    def run_test(self):
+        """
+        There isn't any test logic needed, if the nodes can't connect that means it is broken,
+        so the test is seeing if a devnet can be started.
+        """
+
+if __name__ == '__main__':
+    ConnectDevnetNodes().main()

test/functional/rpc_bind.py

@@ -55,7 +55,7 @@ def run_allowip_test(self, allow_ips, rpchost, rpcport):
         self.nodes[0].rpchost = None
         self.start_nodes([node_args])
         # connect to node through non-loopback interface
-        node = get_rpc_proxy(rpc_url(self.nodes[0].datadir, 0, "%s:%d" % (rpchost, rpcport)), 0, coveragedir=self.options.coveragedir)
+        node = get_rpc_proxy(rpc_url(self.nodes[0].datadir, 0, self.chain, "%s:%d" % (rpchost, rpcport)), 0, coveragedir=self.options.coveragedir)
         node.getnetworkinfo()
         self.stop_nodes()
 

test/functional/test_framework/mininode.py

@@ -96,7 +96,7 @@ def __init__(self):
     def is_connected(self):
         return self._conn_open
 
-    def peer_connect(self, dstaddr, dstport, net="regtest", devnet_name=None):
+    def peer_connect(self, dstaddr, dstport, *, net, devnet_name=None):
         self.dstaddr = dstaddr
         self.dstport = dstport
         self.create_socket(socket.AF_INET, socket.SOCK_STREAM)

test/functional/test_framework/test_framework.py

@@ -49,6 +49,7 @@
     sync_blocks,
     sync_mempools,
     wait_until,
+    get_chain_folder,
 )
 
 class TestStatus(Enum):
@@ -80,6 +81,7 @@ class BitcoinTestFramework():
 
     def __init__(self):
         """Sets test framework defaults. Do not override this method. Instead, override the set_test_params() method"""
+        self.chain = 'regtest'
         self.setup_clean_chain = False
         self.nodes = []
         self.mocktime = 0
@@ -279,7 +281,7 @@ def add_nodes(self, num_nodes, extra_args=None, rpchost=None, timewait=None, bin
         assert_equal(len(binary), num_nodes)
         old_num_nodes = len(self.nodes)
         for i in range(num_nodes):
-            self.nodes.append(TestNode(old_num_nodes + i, get_datadir_path(self.options.tmpdir, old_num_nodes + i), self.extra_args_from_options, rpchost=rpchost, timewait=timewait, bitcoind=binary[i], bitcoin_cli=self.options.bitcoincli, stderr=stderr, mocktime=self.mocktime, coverage_dir=self.options.coveragedir, extra_conf=extra_confs[i], extra_args=extra_args[i], use_cli=self.options.usecli))
+            self.nodes.append(TestNode(old_num_nodes + i, get_datadir_path(self.options.tmpdir, old_num_nodes + i), self.extra_args_from_options, chain=self.chain, rpchost=rpchost, timewait=timewait, bitcoind=binary[i], bitcoin_cli=self.options.bitcoincli, stderr=stderr, mocktime=self.mocktime, coverage_dir=self.options.coveragedir, extra_conf=extra_confs[i], extra_args=extra_args[i], use_cli=self.options.usecli))
 
     def start_node(self, i, *args, **kwargs):
         """Start a dashd"""
@@ -444,13 +446,13 @@ def _initialize_chain(self, extra_args=None, stderr=None):
             # Create cache directories, run dashds:
             self.set_genesis_mocktime()
             for i in range(MAX_NODES):
-                datadir = initialize_datadir(self.options.cachedir, i)
+                datadir = initialize_datadir(self.options.cachedir, i, self.chain)
                 args = [self.options.bitcoind, "-datadir=" + datadir, "-mocktime="+str(GENESISTIME)]
                 if i > 0:
                     args.append("-connect=127.0.0.1:" + str(p2p_port(0)))
                 if extra_args is not None:
                     args.extend(extra_args)
-                self.nodes.append(TestNode(i, get_datadir_path(self.options.cachedir, i), extra_conf=["bind=127.0.0.1"], extra_args=[],extra_args_from_options=self.extra_args_from_options, rpchost=None, timewait=None, bitcoind=self.options.bitcoind, bitcoin_cli=self.options.bitcoincli, stderr=stderr, mocktime=self.mocktime, coverage_dir=None))
+                self.nodes.append(TestNode(i, get_datadir_path(self.options.cachedir, i), chain=self.chain, extra_conf=["bind=127.0.0.1"], extra_args=[],extra_args_from_options=self.extra_args_from_options, rpchost=None, timewait=None, bitcoind=self.options.bitcoind, bitcoin_cli=self.options.bitcoincli, stderr=stderr, mocktime=self.mocktime, coverage_dir=None))
                 self.nodes[i].args = args
                 self.start_node(i)
 
@@ -481,7 +483,8 @@ def _initialize_chain(self, extra_args=None, stderr=None):
             self.disable_mocktime()
 
             def cache_path(n, *paths):
-                return os.path.join(get_datadir_path(self.options.cachedir, n), "regtest", *paths)
+                chain = get_chain_folder(get_datadir_path(self.options.cachedir, n), self.chain)
+                return os.path.join(get_datadir_path(self.options.cachedir, n), chain, *paths)
 
             for i in range(MAX_NODES):
                 for entry in os.listdir(cache_path(i)):
@@ -492,15 +495,15 @@ def cache_path(n, *paths):
             from_dir = get_datadir_path(self.options.cachedir, i)
             to_dir = get_datadir_path(self.options.tmpdir, i)
             shutil.copytree(from_dir, to_dir)
-            initialize_datadir(self.options.tmpdir, i)  # Overwrite port/rpcport in dash.conf
+            initialize_datadir(self.options.tmpdir, i, self.chain)  # Overwrite port/rpcport in dash.conf
 
     def _initialize_chain_clean(self):
         """Initialize empty blockchain for use by the test.
 
         Create an empty blockchain and num_nodes wallets.
         Useful if a test case wants complete control over initialization."""
         for i in range(self.num_nodes):
-            initialize_datadir(self.options.tmpdir, i)
+            initialize_datadir(self.options.tmpdir, i, self.chain)
 
 MASTERNODE_COLLATERAL = 1000
 
@@ -642,7 +645,7 @@ def prepare_datadirs(self):
 
         start_idx = len(self.nodes)
         for idx in range(0, self.mn_count):
-            copy_datadir(0, idx + start_idx, self.options.tmpdir)
+            copy_datadir(0, idx + start_idx, self.options.tmpdir, self.chain)
 
         # restart faucet node
         self.start_node(0)

test/functional/test_framework/test_node.py

@@ -26,6 +26,7 @@
     rpc_url,
     wait_until,
     p2p_port,
+    get_chain_folder,
 )
 
 # For Python 3.4 compatibility
@@ -52,9 +53,10 @@ class TestNode():
     To make things easier for the test writer, any unrecognised messages will
     be dispatched to the RPC connection."""
 
-    def __init__(self, i, datadir, extra_args_from_options, rpchost, timewait, bitcoind, bitcoin_cli, stderr, mocktime, coverage_dir, extra_conf=None, extra_args=None, use_cli=False):
+    def __init__(self, i, datadir, extra_args_from_options, chain, rpchost, timewait, bitcoind, bitcoin_cli, stderr, mocktime, coverage_dir, extra_conf=None, extra_args=None, use_cli=False):
         self.index = i
         self.datadir = datadir
+        self.chain = chain
         self.rpchost = rpchost
         if timewait:
             self.rpc_timeout = timewait
@@ -128,7 +130,7 @@ def start(self, extra_args=None, stderr=None, *args, **kwargs):
         # Delete any existing cookie file -- if such a file exists (eg due to
         # unclean shutdown), it will get overwritten anyway by dashd, and
         # potentially interfere with our attempt to authenticate
-        delete_cookie_file(self.datadir)
+        delete_cookie_file(self.datadir, self.chain)
         self.process = subprocess.Popen(all_args, stderr=stderr, *args, **kwargs)
         self.running = True
         self.log.debug("dashd started, waiting for RPC to come up")
@@ -142,7 +144,7 @@ def wait_for_rpc_connection(self):
                 raise FailedToStartError(self._node_msg(
                     'dashd exited with status {} during initialization'.format(self.process.returncode)))
             try:
-                self.rpc = get_rpc_proxy(rpc_url(self.datadir, self.index, self.rpchost), self.index, timeout=self.rpc_timeout, coveragedir=self.coverage_dir)
+                self.rpc = get_rpc_proxy(rpc_url(self.datadir, self.index, self.chain, self.rpchost), self.index, timeout=self.rpc_timeout, coveragedir=self.coverage_dir)
                 self.rpc.getblockcount()
                 # If the call to getblockcount() succeeds then the RPC connection is up
                 self.rpc_connected = True
@@ -208,7 +210,8 @@ def wait_until_stopped(self, timeout=BITCOIND_PROC_WAIT_TIMEOUT):
 
     @contextlib.contextmanager
     def assert_debug_log(self, expected_msgs):
-        debug_log = os.path.join(self.datadir, 'regtest', 'debug.log')
+        chain = get_chain_folder(self.datadir, self.chain)
+        debug_log = os.path.join(self.datadir, chain, 'debug.log')
         with open(debug_log, encoding='utf-8') as dl:
             dl.seek(0, 2)
             prev_size = dl.tell()
@@ -278,7 +281,7 @@ def add_p2p_connection(self, p2p_conn, *args, **kwargs):
         if 'dstaddr' not in kwargs:
             kwargs['dstaddr'] = '127.0.0.1'
 
-        p2p_conn.peer_connect(*args, **kwargs)
+        p2p_conn.peer_connect(*args, **kwargs, net=self.chain)
         self.p2ps.append(p2p_conn)
 
         return p2p_conn