Merge bitcoin#10798: [tests] [utils] test bitcoin-cli

c6ec435 [tests] Add bitcoin_cli.py test script (John Newbery)
b23549f [tests] add TestNodeCLI class for calling bitcoin-cli for a node (John Newbery)

Pull request description:

  We don't test bitcoin-cli at all. That means that we can miss inconsistencies between the bitcoin-cli client and the RPC interface, such as bitcoin#10698 and bitcoin#10747. It also means that the various bitcoin-cli options and features are untested and regressions could be silently introduced.

  Let's fix that.

  This PR adds bitcoin-cli testing in the python functional test_framework:

  1. Add a bitcoin_cli.py test script that tests bitcoin-cli. At the moment it only tests that the result of `getinfo` is the same if you run it as an RPC or through bitcoin-cli, but can easily be extended to test additional bitcoin-cli features

  **EDIT: `--usecli` option is moved to a separate PR. This PR now only covers the bitcoin_cli.py test.**

  2. ~Add a `--usecli` option to the test framework. This changes the test to use bitcoin-cli for all RPC calls instead of using direct HTTP requests. This is somewhat experimental. It works for most tests, but there are some cases where it can't work transparently because:~
  - ~the testcase is asserting on a specific error code, and bitcoin-cli returns a different error code from the direct RPC~
  - ~we're sending a very large RPC request (eg `submitblock`) and it can't be serialized into a shell bitcoin-cli call.~

  ~I think that even though `--usecli` doesn't work on all tests, it's still a useful experimental feature. Future potential enhancements:~
  - ~enhance the framework to automatically skip tests that are known to fail with bitcoin-cli if the `--usecli` option is used.~
  - ~run a subset of tests in Travis with `-usecli`~

  This builds on and requires the `TestNode` PR bitcoin#10711 . As an aside, this is a good demonstration of how tidy it is to add additional features/interfaces now that test node logic/state is encapsulated in a TestNode class.

  Addresses bitcoin#10791

Tree-SHA512: a1e6be12e8e007f6f67b3d3bbcd142d835787300831eb38e6027a1ad25ca9d79c4bc99a41b19e31ee95205cba1b3b2d21a688b5909316aad70bfc2b4eb6d8a52

test/functional/bitcoin_cli.py

@@ -0,0 +1,26 @@
+#!/usr/bin/env python3
+# Copyright (c) 2017 The Bitcoin Core developers
+# Distributed under the MIT software license, see the accompanying
+# file COPYING or http://www.opensource.org/licenses/mit-license.php.
+"""Test bitcoin-cli"""
+from test_framework.test_framework import BitcoinTestFramework
+from test_framework.util import assert_equal
+
+class TestBitcoinCli(BitcoinTestFramework):
+
+    def __init__(self):
+        super().__init__()
+        self.setup_clean_chain = True
+        self.num_nodes = 1
+
+    def run_test(self):
+        """Main test logic"""
+
+        self.log.info("Compare responses from getinfo RPC and `bitcoin-cli getinfo`")
+        cli_get_info = self.nodes[0].cli.getinfo()
+        rpc_get_info = self.nodes[0].getinfo()
+
+        assert_equal(cli_get_info, rpc_get_info)
+
+if __name__ == '__main__':
+    TestBitcoinCli().main()

test/functional/test_framework/test_node.py

@@ -4,8 +4,10 @@
 # file COPYING or http://www.opensource.org/licenses/mit-license.php.
 """Class for bitcoind node under test"""
 
+import decimal
 import errno
 import http.client
+import json
 import logging
 import os
 import subprocess
@@ -49,6 +51,8 @@ def __init__(self, i, dirname, extra_args, rpchost, timewait, binary, stderr, mo
         self.extra_args = extra_args
         self.args = [self.binary, "-datadir=" + self.datadir, "-server", "-keypool=1", "-discover=0", "-rest", "-logtimemicros", "-debug", "-debugexclude=libevent", "-debugexclude=leveldb", "-mocktime=" + str(mocktime), "-uacomment=testnode%d" % i]
 
+        self.cli = TestNodeCLI(os.getenv("BITCOINCLI", "bitcoin-cli"), self.datadir)
+
         self.running = False
         self.process = None
         self.rpc_connected = False
@@ -136,3 +140,28 @@ def node_encrypt_wallet(self, passphrase):
             time.sleep(0.1)
         self.rpc = None
         self.rpc_connected = False
+
+class TestNodeCLI():
+    """Interface to bitcoin-cli for an individual node"""
+
+    def __init__(self, binary, datadir):
+        self.binary = binary
+        self.datadir = datadir
+
+    def __getattr__(self, command):
+        def dispatcher(*args, **kwargs):
+            return self.send_cli(command, *args, **kwargs)
+        return dispatcher
+
+    def send_cli(self, command, *args, **kwargs):
+        """Run bitcoin-cli command. Deserializes returned string as python object."""
+
+        pos_args = [str(arg) for arg in args]
+        named_args = [str(key) + "=" + str(value) for (key, value) in kwargs.items()]
+        assert not (pos_args and named_args), "Cannot use positional arguments and named arguments in the same bitcoin-cli call"
+        p_args = [self.binary, "-datadir=" + self.datadir]
+        if named_args:
+            p_args += ["-named"]
+        p_args += [command] + pos_args + named_args
+        cli_output = subprocess.check_output(p_args, universal_newlines=True)
+        return json.loads(cli_output, parse_float=decimal.Decimal)

test/functional/test_runner.py

@@ -81,6 +81,7 @@
     # vv Tests less than 30s vv
     'keypool-topup.py',
     'zmq_test.py',
+    'bitcoin_cli.py',
     'mempool_resurrect_test.py',
     'txn_doublespend.py --mineblock',
     'txn_clone.py',
@@ -279,6 +280,7 @@ def run_tests(test_list, src_dir, build_dir, exeext, tmpdir, jobs=1, enable_cove
     #Set env vars
     if "BITCOIND" not in os.environ:
         os.environ["BITCOIND"] = build_dir + '/src/bitcoind' + exeext
+        os.environ["BITCOINCLI"] = build_dir + '/src/bitcoin-cli' + exeext
 
     tests_dir = src_dir + '/test/functional/'