Backport missing PRs from Bitcoin 0.14 - Part 8

.gitignore

@@ -62,7 +62,6 @@ libconftest.dylib*
 *.o
 *.o-*
 *.patch
-.dash
 *.a
 *.pb.cc
 *.pb.h

.travis.yml

@@ -94,6 +94,7 @@ script:
     - mkdir build && cd build
     - ../configure $BITCOIN_CONFIG_ALL $BITCOIN_CONFIG || ( cat config.log && false)
     - make $MAKEJOBS $GOAL || ( echo "Build failure. Verbose build follows." && make $GOAL V=1 ; false )
+    - make distdir
     - export LD_LIBRARY_PATH=$TRAVIS_BUILD_DIR/depends/$HOST/lib
     - if [ "$RUN_TESTS" = "true" ]; then travis_wait 30 make $MAKEJOBS check VERBOSE=1; fi
     - if [ "$RUN_TESTS" = "true" ]; then qa/pull-tester/rpc-tests.py --coverage; fi

Makefile.am

@@ -43,8 +43,7 @@ DIST_DOCS = $(wildcard doc/*.md) $(wildcard doc/release-notes/*.md)
 DIST_CONTRIB = $(top_srcdir)/contrib/dash-cli.bash-completion \
 	       $(top_srcdir)/contrib/dash-tx.bash-completion \
 	       $(top_srcdir)/contrib/dashd.bash-completion \
-	       $(top_srcdir)/contrib/init \
-	       $(top_srcdir)/contrib/rpm
+	       $(top_srcdir)/contrib/init
 
 BIN_CHECKS=$(top_srcdir)/contrib/devtools/symbol-check.py \
            $(top_srcdir)/contrib/devtools/security-check.py
@@ -112,9 +111,16 @@ osx_volname:
 	echo $(OSX_VOLNAME) >$@
 
 if BUILD_DARWIN
-$(OSX_DMG): $(OSX_APP_BUILT) $(OSX_PACKAGING)
+$(OSX_DMG): $(OSX_APP_BUILT) $(OSX_PACKAGING) $(OSX_BACKGROUND_IMAGE)
 	$(PYTHON) $(OSX_DEPLOY_SCRIPT) $(OSX_APP) -add-qt-tr $(OSX_QT_TRANSLATIONS) -translations-dir=$(QT_TRANSLATION_DIR) -dmg -fancy $(OSX_FANCY_PLIST) -verbose 2 -volname $(OSX_VOLNAME)
 
+$(OSX_BACKGROUND_IMAGE).png: contrib/macdeploy/$(OSX_BACKGROUND_SVG)
+	sed 's/PACKAGE_NAME/$(PACKAGE_NAME)/' < "$<" | $(RSVG_CONVERT) -f png -d 36 -p 36 -o $@
+$(OSX_BACKGROUND_IMAGE)@2x.png: contrib/macdeploy/$(OSX_BACKGROUND_SVG)
+	sed 's/PACKAGE_NAME/$(PACKAGE_NAME)/' < "$<" | $(RSVG_CONVERT) -f png -d 72 -p 72 -o $@
+$(OSX_BACKGROUND_IMAGE): $(OSX_BACKGROUND_IMAGE).png $(OSX_BACKGROUND_IMAGE)@2x.png
+	tiffutil -cathidpicheck $^ -out $@
+
 deploydir: $(OSX_DMG)
 else
 APP_DIST_DIR=$(top_builddir)/dist

configure.ac

@@ -320,6 +320,7 @@ case $host in
          fi
        fi
 
+       AC_PATH_PROGS([RSVG_CONVERT], [rsvg-convert rsvg],rsvg-convert)
        AC_CHECK_PROG([BREW],brew, brew)
        if test x$BREW = xbrew; then
          dnl These Homebrew packages may be keg-only, meaning that they won't be found
@@ -415,6 +416,8 @@ if test x$use_lcov = xyes; then
     AC_MSG_ERROR("lcov testing requested but genhtml not found")
   fi
   LCOV="$LCOV --gcov-tool=$GCOV"
+  AX_CHECK_LINK_FLAG([[--coverage]], [LDFLAGS="$LDFLAGS --coverage"],
+    [AC_MSG_ERROR("lcov testing requested but --coverage linker flag does not work")])
   AX_CHECK_COMPILE_FLAG([--coverage],[CXXFLAGS="$CXXFLAGS --coverage"],
     [AC_MSG_ERROR("lcov testing requested but --coverage flag does not work")])
 fi
@@ -833,6 +836,13 @@ else
   fi
 fi
 
+save_CXXFLAGS="${CXXFLAGS}"
+CXXFLAGS="${CXXFLAGS} ${CRYPTO_CFLAGS} ${SSL_CFLAGS}"
+AC_CHECK_DECLS([EVP_MD_CTX_new],,,[AC_INCLUDES_DEFAULT
+#include <openssl/x509_vfy.h>
+])
+CXXFLAGS="${save_CXXFLAGS}"
+
 dnl univalue check
 
 need_bundled_univalue=yes

contrib/debian/examples/dash.conf

@@ -67,9 +67,30 @@
 # This option can be specified multiple times (default: bind to all interfaces)
 #rpcbind=<addr>
 
-# You must set rpcuser and rpcpassword to secure the JSON-RPC api
+# If no rpcpassword is set, rpc cookie auth is sought. The default `-rpccookiefile` name
+# is .cookie and found in the `-datadir` being used for dashd. This option is typically used
+# when the server and client are run as the same user.
+#
+# If not, you must set rpcuser and rpcpassword to secure the JSON-RPC api. The first
+# method(DEPRECATED) is to set this pair for the server and client:
 #rpcuser=Ulysseys
 #rpcpassword=YourSuperGreatPasswordNumber_DO_NOT_USE_THIS_OR_YOU_WILL_GET_ROBBED_385593
+#
+# The second method `rpcauth` can be added to server startup argument. It is set at intialization time
+# using the output from the script in share/rpcuser/rpcuser.py after providing a username:
+#
+# ./share/rpcuser/rpcuser.py alice
+# String to be appended to dash.conf:
+# rpcauth=alice:f7efda5c189b999524f151318c0c86$d5b51b3beffbc02b724e5d095828e0bc8b2456e9ac8757ae3211a5d9b16a22ae
+# Your password:
+# DONT_USE_THIS_YOU_WILL_GET_ROBBED_8ak1gI25KFTvjovL3gAM967mies3E=
+#
+# On client-side, you add the normal user/password pair to send commands:
+#rpcuser=alice
+#rpcpassword=DONT_USE_THIS_YOU_WILL_GET_ROBBED_8ak1gI25KFTvjovL3gAM967mies3E=
+#
+# You can even add multiple entries of these to the server conf file, and client can use any of them:
+# rpcauth=bob:b2dd077cb54591a2f3139e69a897ac$4e71f08d48b4347cf8eff3815c0e25ae2e9a4340474079f55705f40574f4ec99
 
 # How many seconds Dash Core will wait for a complete RPC HTTP request.
 # after the HTTP connection is established. 

contrib/linearize/README.md

@@ -1,5 +1,6 @@
 # Linearize
-Construct a linear, no-fork, best version of the blockchain.
+Construct a linear, no-fork, best version of the Dash blockchain. The scripts
+run using Python 3 but are compatible with Python 2.
 
 ## Step 0: Install dash_hash
 
@@ -10,27 +11,42 @@ https://github.com/dashpay/dash_hash
     $ ./linearize-hashes.py linearize.cfg > hashlist.txt
 
 Required configuration file settings for linearize-hashes:
-* RPC: rpcuser, rpcpassword
+* RPC: `rpcuser`, `rpcpassword`
 
 Optional config file setting for linearize-hashes:
-* RPC: host, port
-* Block chain: min_height, max_height
+* RPC: `host`  (Default: `127.0.0.1`)
+* RPC: `port`  (Default: `9998`)
+* Blockchain: `min_height`, `max_height`
+* `rev_hash_bytes`: If true, the written block hash list will be
+byte-reversed. (In other words, the hash returned by getblockhash will have its
+bytes reversed.) False by default. Intended for generation of
+standalone hash lists but safe to use with linearize-data.py, which will output
+the same data no matter which byte format is chosen.
+
+The `linearize-hashes` script requires a connection, local or remote, to a
+JSON-RPC server. Running `dashd` or `dash-qt -server` will be sufficient.
 
 ## Step 2: Copy local block data
 
     $ ./linearize-data.py linearize.cfg
 
 Required configuration file settings:
-* "input": bitcoind blocks/ directory containing blkNNNNN.dat
-* "hashlist": text file containing list of block hashes, linearized-hashes.py
-output.
-* "output_file" for bootstrap.dat or "output" for output directory for linearized blocks/blkNNNNN.dat output
+* `output_file`: The file that will contain the final blockchain.
+      or
+* `output`: Output directory for linearized `blocks/blkNNNNN.dat` output.
 
 Optional config file setting for linearize-data:
-* "netmagic": network magic number (default is 'cee2caff', testnet)
-* "genesis": genesis block hash (default is '00000bafbc94add76cb75e2ec92894837288a481e5c005f6563d91623bf8bc2c', testnet)
-* "max_out_sz": maximum output file size (default 100 \* 1000 \* 1000)
-* "split_timestamp": Split files when a new month is first seen, in addition to
-reaching a maximum file size.
-* "file_timestamp": Set each file's last-modified time to that of the
-most recent block in that file.
+* `file_timestamp`: Set each file's last-modified time to that of the most
+recent block in that file.
+* `genesis`: The hash of the genesis block in the blockchain. (default is '00000ffd590b1485b3caadc19b22e6379c733355108f107a430458cdf3407ab6', mainnet)
+* `input`: dashd blocks/ directory containing blkNNNNN.dat
+* `hashlist`: text file containing list of block hashes created by
+linearize-hashes.py.
+* `max_out_sz`: Maximum size for files created by the `output_file` option.
+(Default: `1000*1000*1000 bytes`)
+* `netmagic`: Network magic number. (default is 'bf0c6bbd', mainnet)
+* `rev_hash_bytes`: If true, the block hash list written by linearize-hashes.py
+will be byte-reversed when read by linearize-data.py. See the linearize-hashes
+entry for more information.
+* `split_timestamp`: Split blockchain files when a new month is first seen, in
+addition to reaching a maximum file size (`max_out_sz`).

contrib/linearize/example-linearize.cfg

@@ -13,8 +13,10 @@ netmagic=bf0c6bbd
 input=/home/example/.dashcore/blocks
 output_file=/home/example/Downloads/bootstrap.dat
 hashlist=hashlist.txt
-split_year=1
 genesis=00000ffd590b1485b3caadc19b22e6379c733355108f107a430458cdf3407ab6
 
 # Maxmimum size in bytes of out-of-order blocks cache in memory
 out_of_order_cache_sz = 10000000
+
+# Do we want the reverse the hash bytes coming from getblockhash?
+rev_hash_bytes = False

contrib/linearize/linearize-data.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 #
 # linearize-data.py: Construct a linear, no-fork version of the chain.
 #
@@ -8,24 +8,34 @@
 #
 
 from __future__ import print_function, division
+try: # Python 3
+    import http.client as httplib
+except ImportError: # Python 2
+    import httplib
 import json
 import struct
 import re
 import os
 import os.path
 import base64
-import httplib
 import sys
 import hashlib
 import dash_hash
 import datetime
 import time
 from collections import namedtuple
+from binascii import hexlify, unhexlify
 
 settings = {}
 
+##### Switch endian-ness #####
+def hex_switchEndian(s):
+	""" Switches the endianness of a hex string (in pairs of hex chars) """
+	pairList = [s[i:i+2].encode() for i in range(0, len(s), 2)]
+	return b''.join(pairList[::-1]).decode()
+
 def uint32(x):
-	return x & 0xffffffffL
+	return x & 0xffffffff
 
 def bytereverse(x):
 	return uint32(( ((x) << 24) | (((x) << 8) & 0x00ff0000) |
@@ -36,14 +46,14 @@ def bufreverse(in_buf):
 	for i in range(0, len(in_buf), 4):
 		word = struct.unpack('@I', in_buf[i:i+4])[0]
 		out_words.append(struct.pack('@I', bytereverse(word)))
-	return ''.join(out_words)
+	return b''.join(out_words)
 
 def wordreverse(in_buf):
 	out_words = []
 	for i in range(0, len(in_buf), 4):
 		out_words.append(in_buf[i:i+4])
 	out_words.reverse()
-	return ''.join(out_words)
+	return b''.join(out_words)
 
 def calc_hdr_hash(blk_hdr):
 	#hash1 = hashlib.sha256()
@@ -62,7 +72,7 @@ def calc_hash_str(blk_hdr):
 	hash = calc_hdr_hash(blk_hdr)
 	hash = bufreverse(hash)
 	hash = wordreverse(hash)
-	hash_str = hash.encode('hex')
+	hash_str = hexlify(hash).decode('utf-8')
 	return hash_str
 
 def get_blk_dt(blk_hdr):
@@ -72,17 +82,21 @@ def get_blk_dt(blk_hdr):
 	dt_ym = datetime.datetime(dt.year, dt.month, 1)
 	return (dt_ym, nTime)
 
+# When getting the list of block hashes, undo any byte reversals.
 def get_block_hashes(settings):
 	blkindex = []
 	f = open(settings['hashlist'], "r")
 	for line in f:
 		line = line.rstrip()
+		if settings['rev_hash_bytes'] == 'true':
+			line = hex_switchEndian(line)
 		blkindex.append(line)
 
 	print("Read " + str(len(blkindex)) + " hashes")
 
 	return blkindex
 
+# The block map shouldn't give or receive byte-reversed hashes.
 def mkblockmap(blkindex):
 	blkmap = {}
 	for height,hash in enumerate(blkindex):
@@ -210,7 +224,7 @@ def run(self):
 
 			inMagic = inhdr[:4]
 			if (inMagic != self.settings['netmagic']):
-				print("Invalid magic: " + inMagic.encode('hex'))
+				print("Invalid magic: " + hexlify(inMagic).decode('utf-8'))
 				return
 			inLenLE = inhdr[4:]
 			su = struct.unpack("<I", inLenLE)
@@ -268,10 +282,16 @@ def run(self):
 		settings[m.group(1)] = m.group(2)
 	f.close()
 
+	# Force hash byte format setting to be lowercase to make comparisons easier.
+	# Also place upfront in case any settings need to know about it.
+	if 'rev_hash_bytes' not in settings:
+		settings['rev_hash_bytes'] = 'false'
+	settings['rev_hash_bytes'] = settings['rev_hash_bytes'].lower()
+
 	if 'netmagic' not in settings:
-		settings['netmagic'] = 'cee2caff'
+		settings['netmagic'] = 'bf0c6bbd'
 	if 'genesis' not in settings:
-		settings['genesis'] = '00000bafbc94add76cb75e2ec92894837288a481e5c005f6563d91623bf8bc2c'
+		settings['genesis'] = '00000ffd590b1485b3caadc19b22e6379c733355108f107a430458cdf3407ab6'
 	if 'input' not in settings:
 		settings['input'] = 'input'
 	if 'hashlist' not in settings:
@@ -281,14 +301,14 @@ def run(self):
 	if 'split_timestamp' not in settings:
 		settings['split_timestamp'] = 0
 	if 'max_out_sz' not in settings:
-		settings['max_out_sz'] = 1000L * 1000 * 1000
+		settings['max_out_sz'] = 1000 * 1000 * 1000
 	if 'out_of_order_cache_sz' not in settings:
 		settings['out_of_order_cache_sz'] = 100 * 1000 * 1000
 
-	settings['max_out_sz'] = long(settings['max_out_sz'])
+	settings['max_out_sz'] = int(settings['max_out_sz'])
 	settings['split_timestamp'] = int(settings['split_timestamp'])
 	settings['file_timestamp'] = int(settings['file_timestamp'])
-	settings['netmagic'] = settings['netmagic'].decode('hex')
+	settings['netmagic'] = unhexlify(settings['netmagic'].encode('utf-8'))
 	settings['out_of_order_cache_sz'] = int(settings['out_of_order_cache_sz'])
 
 	if 'output_file' not in settings and 'output' not in settings:
@@ -298,8 +318,8 @@ def run(self):
 	blkindex = get_block_hashes(settings)
 	blkmap = mkblockmap(blkindex)
 
+	# Block hash map won't be byte-reversed. Neither should the genesis hash.
 	if not settings['genesis'] in blkmap:
 		print("Genesis block not found in hashlist")
 	else:
 		BlockDataCopier(settings, blkindex, blkmap).run()
-