diff --git a/.gitignore b/.gitignore index bfaa1ed3..712da333 100644 --- a/.gitignore +++ b/.gitignore @@ -6,3 +6,6 @@ blockchain.cache env htmlcov/ joinmarket.cfg +cmttools/commitments.json +blacklist + diff --git a/.travis.yml b/.travis.yml index 69b24d3b..7c5acd92 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,3 +1,5 @@ +sudo: required +dist: trusty language: python python: - "2.7_with_system_site_packages" @@ -27,11 +29,13 @@ script: - cd .. #set up joinmarket.cfg - cp test/regtest_joinmarket.cfg joinmarket.cfg +#install miniircd + - git clone git://github.com/Joinmarket-Org/miniircd.git #setup bitcoin config file - mkdir /home/travis/.bitcoin - cp test/bitcoin.conf /home/travis/.bitcoin/. - chmod 600 /home/travis/.bitcoin/bitcoin.conf - - PYTHONPATH=.:$PYTHONPATH py.test --cov-report html --btcconf=/home/travis/.bitcoin/bitcoin.conf --btcpwd=123456abcdef --ignore test/test_tumbler.py + - PYTHONPATH=.:$PYTHONPATH py.test --cov=joinmarket/ --cov=test/ --cov=bitcoin/ --cov-report html --nirc=2 --btcconf=/home/travis/.bitcoin/bitcoin.conf --btcuser=bitcoinrpc --btcpwd=123456abcdef --ignore test/test_tumbler.py - cd logs - for x in `ls`; do tail -50 $x; done - cd .. diff --git a/README.md b/README.md index 339c1fdd..5677e99b 100644 --- a/README.md +++ b/README.md @@ -14,6 +14,9 @@ Widespread use of JoinMarket could improve bitcoin's fungibility as a commodity. ##Installation +#####A NOTE ON UPDATING +The installation is slightly changed, with the secp256k1 python binding no longer being optional, and libnacl now being installed via pip, not locally. The short version is: do follow the below process, for example the secp256k1 binding must be the latest version else you'll get errors. Of course if you already have libsodium you don't need to re-install it. + #####REQUIRED INSTALLATION DEPENDENCIES + You will need python 2.7 @@ -33,22 +36,39 @@ Widespread use of JoinMarket could improve bitcoin's fungibility as a commodity. make check sudo make install ``` -+ (optional, recommended): Install the libsecp256k1 Python library: + ++ Install Python dependencies: + You need `pip` (e.g. `sudo apt-get python-pip`, recent Python usually ships with pip). + + (Setuptools version must be >= 3.3). + + (Recommended but not needed) : use `virtualenv` to keep dependencies isolated. + + If on Linux/OSX: + + The Python binding to libsecp256k1 will most likely have some dependencies; read the [Wiki article](https://github.com/JoinMarket-Org/joinmarket/wiki/Installing-the-libsecp256k1-binding). + + Try running the below without following those instructions, most likely it will fail and you will then have to follow them. + + ``` + pip install -r requirements.txt + ``` + + If on Windows: ``` - pip install secp256k1 + pip install -r requirements-windows.txt ``` - Note that this requires `pip`. This library will make use of libsecp256k1 if you already have it installed on your system. Most likely you won't, and it will try to build libsecp256k1 automatically for you. This requires some development packages; please read the installation details [here](https://github.com/ludbb/secp256k1-py#installation), provided for Debian/Ubuntu and OS X. Please also note that if you can't complete this step, Joinmarket will still run correctly, with two disadvantages: wallet loading is much slower without libsecp256k1, and the ECC crypto code is far less robust and well tested. + Matplotlib for displaying the graphs in orderbook-watcher (optional) ###DEBIAN / UBUNTU QUICK INSTALL FOR USERS: 1. `sudo apt-get update -y && sudo apt-get upgrade -y && sudo apt-get install python libsodium-dev -y` -2. `pip install secp256k1` (optional but recommended) +2. `pip install -r requirements.txt` 2. `sudo apt-get install python-matplotlib -y` (optional) -3. Download JoinMarket 0.1.4 source from [here](https://github.com/joinmarket-org/joinmarket/releases/tag/v0.1.4) -4. Extract or unzip and `cd joinmarket-0.1.4` +3. Download JoinMarket 0.2.0 source from [here](https://github.com/joinmarket-org/joinmarket/releases/tag/v0.2.0) +4. Extract or unzip and `cd joinmarket-0.2.0` 4. Generating your first wallet will populate the configuration file: `joinmarket.cfg`. Check if the default settings suit your needs. diff --git a/bitcoin/__init__.py b/bitcoin/__init__.py index 3ca1223a..5acc8598 100644 --- a/bitcoin/__init__.py +++ b/bitcoin/__init__.py @@ -1,14 +1,8 @@ from bitcoin.py2specials import * from bitcoin.py3specials import * -secp_present = False -try: - import secp256k1 - secp_present = True - from bitcoin.secp256k1_main import * - from bitcoin.secp256k1_transaction import * - from bitcoin.secp256k1_deterministic import * -except ImportError as e: - from bitcoin.main import * - from bitcoin.deterministic import * - from bitcoin.transaction import * +import secp256k1 +from bitcoin.secp256k1_main import * +from bitcoin.secp256k1_transaction import * +from bitcoin.secp256k1_deterministic import * from bitcoin.bci import * +from bitcoin.podle import * diff --git a/bitcoin/deterministic.py b/bitcoin/deterministic.py deleted file mode 100644 index d74f6cc3..00000000 --- a/bitcoin/deterministic.py +++ /dev/null @@ -1,128 +0,0 @@ -from bitcoin.main import * -import hmac -import hashlib -from binascii import hexlify - -# Below code ASSUMES binary inputs and compressed pubkeys -MAINNET_PRIVATE = b'\x04\x88\xAD\xE4' -MAINNET_PUBLIC = b'\x04\x88\xB2\x1E' -TESTNET_PRIVATE = b'\x04\x35\x83\x94' -TESTNET_PUBLIC = b'\x04\x35\x87\xCF' -PRIVATE = [MAINNET_PRIVATE, TESTNET_PRIVATE] -PUBLIC = [MAINNET_PUBLIC, TESTNET_PUBLIC] - -# BIP32 child key derivation - -def raw_bip32_ckd(rawtuple, i): - vbytes, depth, fingerprint, oldi, chaincode, key = rawtuple - i = int(i) - - if vbytes in PRIVATE: - priv = key - pub = privtopub(key) - else: - pub = key - - if i >= 2**31: - if vbytes in PUBLIC: - raise Exception("Can't do private derivation on public key!") - I = hmac.new(chaincode, b'\x00' + priv[:32] + encode(i, 256, 4), - hashlib.sha512).digest() - else: - I = hmac.new(chaincode, pub + encode(i, 256, 4), - hashlib.sha512).digest() - - if vbytes in PRIVATE: - newkey = add_privkeys(I[:32] + B'\x01', priv) - fingerprint = bin_hash160(privtopub(key))[:4] - if vbytes in PUBLIC: - newkey = add_pubkeys(compress(privtopub(I[:32])), key) - fingerprint = bin_hash160(key)[:4] - - return (vbytes, depth + 1, fingerprint, i, I[32:], newkey) - - -def bip32_serialize(rawtuple): - vbytes, depth, fingerprint, i, chaincode, key = rawtuple - i = encode(i, 256, 4) - chaincode = encode(hash_to_int(chaincode), 256, 32) - keydata = b'\x00' + key[:-1] if vbytes in PRIVATE else key - bindata = vbytes + from_int_to_byte( - depth % 256) + fingerprint + i + chaincode + keydata - return changebase(bindata + bin_dbl_sha256(bindata)[:4], 256, 58) - - -def bip32_deserialize(data): - dbin = changebase(data, 58, 256) - if bin_dbl_sha256(dbin[:-4])[:4] != dbin[-4:]: - raise Exception("Invalid checksum") - vbytes = dbin[0:4] - depth = from_byte_to_int(dbin[4]) - fingerprint = dbin[5:9] - i = decode(dbin[9:13], 256) - chaincode = dbin[13:45] - key = dbin[46:78] + b'\x01' if vbytes in PRIVATE else dbin[45:78] - return (vbytes, depth, fingerprint, i, chaincode, key) - - -def raw_bip32_privtopub(rawtuple): - vbytes, depth, fingerprint, i, chaincode, key = rawtuple - newvbytes = MAINNET_PUBLIC if vbytes == MAINNET_PRIVATE else TESTNET_PUBLIC - return (newvbytes, depth, fingerprint, i, chaincode, privtopub(key)) - - -def bip32_privtopub(data): - return bip32_serialize(raw_bip32_privtopub(bip32_deserialize(data))) - - -def bip32_ckd(data, i): - return bip32_serialize(raw_bip32_ckd(bip32_deserialize(data), i)) - - -def bip32_master_key(seed, vbytes=MAINNET_PRIVATE): - I = hmac.new( - from_string_to_bytes("Bitcoin seed"), seed, hashlib.sha512).digest() - return bip32_serialize((vbytes, 0, b'\x00' * 4, 0, I[32:], I[:32] + b'\x01' - )) - - -def bip32_bin_extract_key(data): - return bip32_deserialize(data)[-1] - - -def bip32_extract_key(data): - return safe_hexlify(bip32_deserialize(data)[-1]) - -# Exploits the same vulnerability as above in Electrum wallets -# Takes a BIP32 pubkey and one of the child privkeys of its corresponding -# privkey and returns the BIP32 privkey associated with that pubkey - -def raw_crack_bip32_privkey(parent_pub, priv): - vbytes, depth, fingerprint, i, chaincode, key = priv - pvbytes, pdepth, pfingerprint, pi, pchaincode, pkey = parent_pub - i = int(i) - - if i >= 2**31: - raise Exception("Can't crack private derivation!") - - I = hmac.new(pchaincode, pkey + encode(i, 256, 4), hashlib.sha512).digest() - - pprivkey = subtract_privkeys(key, I[:32] + b'\x01') - - newvbytes = MAINNET_PRIVATE if vbytes == MAINNET_PUBLIC else TESTNET_PRIVATE - return (newvbytes, pdepth, pfingerprint, pi, pchaincode, pprivkey) - - -def crack_bip32_privkey(parent_pub, priv): - dsppub = bip32_deserialize(parent_pub) - dspriv = bip32_deserialize(priv) - return bip32_serialize(raw_crack_bip32_privkey(dsppub, dspriv)) - -def bip32_descend(*args): - if len(args) == 2: - key, path = args - else: - key, path = args[0], map(int, args[1:]) - for p in path: - key = bip32_ckd(key, p) - return bip32_extract_key(key) diff --git a/bitcoin/main.py b/bitcoin/main.py deleted file mode 100644 index e3ccb4e0..00000000 --- a/bitcoin/main.py +++ /dev/null @@ -1,504 +0,0 @@ -#!/usr/bin/python -from .py2specials import * -from .py3specials import * -import binascii -import hashlib -import re -import sys -import os -import base64 -import time -import random -import hmac - -is_python2 = sys.version_info.major == 2 - -# Elliptic curve parameters (secp256k1) - -P = 2**256 - 2**32 - 977 -N = 115792089237316195423570985008687907852837564279074904382605163141518161494337 -A = 0 -B = 7 -Gx = 55066263022277343669578718895168534326250603453777594175500187360389116729240 -Gy = 32670510020758816978083085130507043184471273380659243275938904335757337482424 -G = (Gx, Gy) - -# Extended Euclidean Algorithm -def inv(a, n): - lm, hm = 1, 0 - low, high = a % n, n - while low > 1: - r = high // low - nm, new = hm - lm * r, high - low * r - lm, low, hm, high = nm, new, lm, low - return lm % n - -# Elliptic curve Jordan form functions -# P = (m, n, p, q) where m/n = x, p/q = y - -def isinf(p): - return p[0] == 0 and p[1] == 0 - - -def jordan_isinf(p): - return p[0][0] == 0 and p[1][0] == 0 - - -def mulcoords(c1, c2): - return (c1[0] * c2[0] % P, c1[1] * c2[1] % P) - - -def mul_by_const(c, v): - return (c[0] * v % P, c[1]) - - -def addcoords(c1, c2): - return ((c1[0] * c2[1] + c2[0] * c1[1]) % P, c1[1] * c2[1] % P) - - -def subcoords(c1, c2): - return ((c1[0] * c2[1] - c2[0] * c1[1]) % P, c1[1] * c2[1] % P) - - -def invcoords(c): - return (c[1], c[0]) - - -def jordan_add(a, b): - if jordan_isinf(a): - return b - if jordan_isinf(b): - return a - - if (a[0][0] * b[0][1] - b[0][0] * a[0][1]) % P == 0: - if (a[1][0] * b[1][1] - b[1][0] * a[1][1]) % P == 0: - return jordan_double(a) - else: - return ((0, 1), (0, 1)) - xdiff = subcoords(b[0], a[0]) - ydiff = subcoords(b[1], a[1]) - m = mulcoords(ydiff, invcoords(xdiff)) - x = subcoords(subcoords(mulcoords(m, m), a[0]), b[0]) - y = subcoords(mulcoords(m, subcoords(a[0], x)), a[1]) - return (x, y) - - -def jordan_double(a): - if jordan_isinf(a): - return ((0, 1), (0, 1)) - num = addcoords(mul_by_const(mulcoords(a[0], a[0]), 3), (A, 1)) - den = mul_by_const(a[1], 2) - m = mulcoords(num, invcoords(den)) - x = subcoords(mulcoords(m, m), mul_by_const(a[0], 2)) - y = subcoords(mulcoords(m, subcoords(a[0], x)), a[1]) - return (x, y) - - -def jordan_multiply(a, n): - if jordan_isinf(a) or n == 0: - return ((0, 0), (0, 0)) - if n == 1: - return a - if n < 0 or n >= N: - return jordan_multiply(a, n % N) - if (n % 2) == 0: - return jordan_double(jordan_multiply(a, n // 2)) - if (n % 2) == 1: - return jordan_add(jordan_double(jordan_multiply(a, n // 2)), a) - - -def to_jordan(p): - return ((p[0], 1), (p[1], 1)) - - -def from_jordan(p): - return (p[0][0] * inv(p[0][1], P) % P, p[1][0] * inv(p[1][1], P) % P) - -def fast_multiply(a, n): - return from_jordan(jordan_multiply(to_jordan(a), n)) - - -def fast_add(a, b): - return from_jordan(jordan_add(to_jordan(a), to_jordan(b))) - -# Functions for handling pubkey and privkey formats - - -def get_pubkey_format(pub): - if is_python2: - two = '\x02' - three = '\x03' - four = '\x04' - else: - two = 2 - three = 3 - four = 4 - - if isinstance(pub, (tuple, list)): return 'decimal' - elif len(pub) == 65 and pub[0] == four: return 'bin' - elif len(pub) == 130 and pub[0:2] == '04': return 'hex' - elif len(pub) == 33 and pub[0] in [two, three]: return 'bin_compressed' - elif len(pub) == 66 and pub[0:2] in ['02', '03']: return 'hex_compressed' - elif len(pub) == 64: return 'bin_electrum' - elif len(pub) == 128: return 'hex_electrum' - else: raise Exception("Pubkey not in recognized format") - - -def encode_pubkey(pub, formt): - if not isinstance(pub, (tuple, list)): - pub = decode_pubkey(pub) - if formt == 'decimal': return pub - elif formt == 'bin': - return b'\x04' + encode(pub[0], 256, 32) + encode(pub[1], 256, 32) - elif formt == 'bin_compressed': - return from_int_to_byte(2 + (pub[1] % 2)) + encode(pub[0], 256, 32) - elif formt == 'hex': - return '04' + encode(pub[0], 16, 64) + encode(pub[1], 16, 64) - elif formt == 'hex_compressed': - return '0' + str(2 + (pub[1] % 2)) + encode(pub[0], 16, 64) - elif formt == 'bin_electrum': - return encode(pub[0], 256, 32) + encode(pub[1], 256, 32) - elif formt == 'hex_electrum': - return encode(pub[0], 16, 64) + encode(pub[1], 16, 64) - else: - raise Exception("Invalid format!") - - -def decode_pubkey(pub, formt=None): - if not formt: formt = get_pubkey_format(pub) - if formt == 'decimal': return pub - elif formt == 'bin': - return (decode(pub[1:33], 256), decode(pub[33:65], 256)) - elif formt == 'bin_compressed': - x = decode(pub[1:33], 256) - beta = pow(int(x * x * x + A * x + B), int((P + 1) // 4), int(P)) - y = (P - beta) if ((beta + from_byte_to_int(pub[0])) % 2) else beta - return (x, y) - elif formt == 'hex': - return (decode(pub[2:66], 16), decode(pub[66:130], 16)) - elif formt == 'hex_compressed': - return decode_pubkey(safe_from_hex(pub), 'bin_compressed') - elif formt == 'bin_electrum': - return (decode(pub[:32], 256), decode(pub[32:64], 256)) - elif formt == 'hex_electrum': - return (decode(pub[:64], 16), decode(pub[64:128], 16)) - else: - raise Exception("Invalid format!") - - -def get_privkey_format(priv): - if isinstance(priv, int_types): return 'decimal' - elif len(priv) == 32: return 'bin' - elif len(priv) == 33: return 'bin_compressed' - elif len(priv) == 64: return 'hex' - elif len(priv) == 66: return 'hex_compressed' - else: - bin_p = b58check_to_bin(priv) - if len(bin_p) == 32: return 'wif' - elif len(bin_p) == 33: return 'wif_compressed' - else: raise Exception("WIF does not represent privkey") - - -def encode_privkey(priv, formt, vbyte=0): - if not isinstance(priv, int_types): - return encode_privkey(decode_privkey(priv), formt, vbyte) - if formt == 'decimal': return priv - elif formt == 'bin': return encode(priv, 256, 32) - elif formt == 'bin_compressed': return encode(priv, 256, 32) + b'\x01' - elif formt == 'hex': return encode(priv, 16, 64) - elif formt == 'hex_compressed': return encode(priv, 16, 64) + '01' - elif formt == 'wif': - return bin_to_b58check(encode(priv, 256, 32), 128 + int(vbyte)) - elif formt == 'wif_compressed': - return bin_to_b58check( - encode(priv, 256, 32) + b'\x01', 128 + int(vbyte)) - else: - raise Exception("Invalid format!") - - -def decode_privkey(priv, formt=None): - if not formt: formt = get_privkey_format(priv) - if formt == 'decimal': return priv - elif formt == 'bin': return decode(priv, 256) - elif formt == 'bin_compressed': return decode(priv[:32], 256) - elif formt == 'hex': return decode(priv, 16) - elif formt == 'hex_compressed': return decode(priv[:64], 16) - elif formt == 'wif': return decode(b58check_to_bin(priv), 256) - elif formt == 'wif_compressed': - return decode(b58check_to_bin(priv)[:32], 256) - else: - raise Exception("WIF does not represent privkey") - - -def add_pubkeys(p1, p2): - f1, f2 = get_pubkey_format(p1), get_pubkey_format(p2) - return encode_pubkey( - fast_add( - decode_pubkey(p1, f1), decode_pubkey(p2, f2)), f1) - - -def add_privkeys(p1, p2): - f1, f2 = get_privkey_format(p1), get_privkey_format(p2) - return encode_privkey( - (decode_privkey(p1, f1) + decode_privkey(p2, f2)) % N, f1) - - -def multiply(pubkey, privkey): - f1, f2 = get_pubkey_format(pubkey), get_privkey_format(privkey) - pubkey, privkey = decode_pubkey(pubkey, f1), decode_privkey(privkey, f2) - # http://safecurves.cr.yp.to/twist.html - if not isinf(pubkey) and ( - pubkey[0]**3 + B - pubkey[1] * pubkey[1]) % P != 0: - raise Exception("Point not on curve") - return encode_pubkey(fast_multiply(pubkey, privkey), f1) - - -def divide(pubkey, privkey): - factor = inv(decode_privkey(privkey), N) - return multiply(pubkey, factor) - - -def compress(pubkey): - f = get_pubkey_format(pubkey) - if 'compressed' in f: return pubkey - elif f == 'bin': - return encode_pubkey(decode_pubkey(pubkey, f), 'bin_compressed') - elif f == 'hex' or f == 'decimal': - return encode_pubkey(decode_pubkey(pubkey, f), 'hex_compressed') - - -def decompress(pubkey): - f = get_pubkey_format(pubkey) - if 'compressed' not in f: return pubkey - elif f == 'bin_compressed': - return encode_pubkey(decode_pubkey(pubkey, f), 'bin') - elif f == 'hex_compressed' or f == 'decimal': - return encode_pubkey(decode_pubkey(pubkey, f), 'hex') - - -def privkey_to_pubkey(privkey): - f = get_privkey_format(privkey) - privkey = decode_privkey(privkey, f) - if privkey >= N: - raise Exception("Invalid privkey") - if f in ['bin', 'bin_compressed', 'hex', 'hex_compressed', 'decimal']: - return encode_pubkey(fast_multiply(G, privkey), f) - else: - return encode_pubkey(fast_multiply(G, privkey), f.replace('wif', 'hex')) - - -privtopub = privkey_to_pubkey - - -def privkey_to_address(priv, magicbyte=0): - return pubkey_to_address(privkey_to_pubkey(priv), magicbyte) - - -privtoaddr = privkey_to_address - - -def neg_pubkey(pubkey): - f = get_pubkey_format(pubkey) - pubkey = decode_pubkey(pubkey, f) - return encode_pubkey((pubkey[0], (P - pubkey[1]) % P), f) - - -def neg_privkey(privkey): - f = get_privkey_format(privkey) - privkey = decode_privkey(privkey, f) - return encode_privkey((N - privkey) % N, f) - - -def subtract_pubkeys(p1, p2): - f1, f2 = get_pubkey_format(p1), get_pubkey_format(p2) - k2 = decode_pubkey(p2, f2) - return encode_pubkey( - fast_add( - decode_pubkey(p1, f1), (k2[0], (P - k2[1]) % P)), f1) - - -def subtract_privkeys(p1, p2): - f1, f2 = get_privkey_format(p1), get_privkey_format(p2) - k2 = decode_privkey(p2, f2) - return encode_privkey((decode_privkey(p1, f1) - k2) % N, f1) - -# Hashes - - -def bin_hash160(string): - intermed = hashlib.sha256(string).digest() - digest = '' - digest = hashlib.new('ripemd160', intermed).digest() - return digest - - -def hash160(string): - return safe_hexlify(bin_hash160(string)) - - -def bin_sha256(string): - binary_data = string if isinstance(string, bytes) else bytes(string, - 'utf-8') - return hashlib.sha256(binary_data).digest() - - -def sha256(string): - return bytes_to_hex_string(bin_sha256(string)) - - -def bin_ripemd160(string): - digest = hashlib.new('ripemd160', string).digest() - return digest - - -def ripemd160(string): - return safe_hexlify(bin_ripemd160(string)) - - -def bin_dbl_sha256(s): - bytes_to_hash = from_string_to_bytes(s) - return hashlib.sha256(hashlib.sha256(bytes_to_hash).digest()).digest() - - -def dbl_sha256(string): - return safe_hexlify(bin_dbl_sha256(string)) - - -def bin_slowsha(string): - string = from_string_to_bytes(string) - orig_input = string - for i in range(100000): - string = hashlib.sha256(string + orig_input).digest() - return string - - -def slowsha(string): - return safe_hexlify(bin_slowsha(string)) - - -def hash_to_int(x): - if len(x) in [40, 64]: - return decode(x, 16) - return decode(x, 256) - - -def num_to_var_int(x): - x = int(x) - if x < 253: return from_int_to_byte(x) - elif x < 65536: return from_int_to_byte(253) + encode(x, 256, 2)[::-1] - elif x < 4294967296: return from_int_to_byte(254) + encode(x, 256, 4)[::-1] - else: return from_int_to_byte(255) + encode(x, 256, 8)[::-1] - - -# WTF, Electrum? -def electrum_sig_hash(message): - padded = b"\x18Bitcoin Signed Message:\n" + num_to_var_int(len( - message)) + from_string_to_bytes(message) - return bin_dbl_sha256(padded) - -# Encodings - -def b58check_to_bin(inp): - leadingzbytes = len(re.match('^1*', inp).group(0)) - data = b'\x00' * leadingzbytes + changebase(inp, 58, 256) - assert bin_dbl_sha256(data[:-4])[:4] == data[-4:] - return data[1:-4] - - -def get_version_byte(inp): - leadingzbytes = len(re.match('^1*', inp).group(0)) - data = b'\x00' * leadingzbytes + changebase(inp, 58, 256) - assert bin_dbl_sha256(data[:-4])[:4] == data[-4:] - return ord(data[0]) - - -def hex_to_b58check(inp, magicbyte=0): - return bin_to_b58check(binascii.unhexlify(inp), magicbyte) - - -def b58check_to_hex(inp): - return safe_hexlify(b58check_to_bin(inp)) - - -def pubkey_to_address(pubkey, magicbyte=0): - if isinstance(pubkey, (list, tuple)): - pubkey = encode_pubkey(pubkey, 'bin') - if len(pubkey) in [66, 130]: - return bin_to_b58check( - bin_hash160(binascii.unhexlify(pubkey)), magicbyte) - return bin_to_b58check(bin_hash160(pubkey), magicbyte) - - -pubtoaddr = pubkey_to_address - -# EDCSA - - -def encode_sig(v, r, s): - vb, rb, sb = from_int_to_byte(v), encode(r, 256), encode(s, 256) - - result = base64.b64encode(vb + b'\x00' * (32 - len(rb)) + rb + b'\x00' * ( - 32 - len(sb)) + sb) - return result if is_python2 else str(result, 'utf-8') - - -def decode_sig(sig): - bytez = base64.b64decode(sig) - return from_byte_to_int(bytez[0]), decode(bytez[1:33], 256), decode( - bytez[33:], 256) - -# https://tools.ietf.org/html/rfc6979#section-3.2 - - -def deterministic_generate_k(msghash, priv): - v = b'\x01' * 32 - k = b'\x00' * 32 - priv = encode_privkey(priv, 'bin') - msghash = encode(hash_to_int(msghash), 256, 32) - k = hmac.new(k, v + b'\x00' + priv + msghash, hashlib.sha256).digest() - v = hmac.new(k, v, hashlib.sha256).digest() - k = hmac.new(k, v + b'\x01' + priv + msghash, hashlib.sha256).digest() - v = hmac.new(k, v, hashlib.sha256).digest() - return decode(hmac.new(k, v, hashlib.sha256).digest(), 256) - - -def ecdsa_raw_sign(msghash, priv): - - z = hash_to_int(msghash) - k = deterministic_generate_k(msghash, priv) - - r, y = fast_multiply(G, k) - s = inv(k, N) * (z + r * decode_privkey(priv)) % N - - return 27 + (y % 2), r, s - - -def ecdsa_sign(msg, priv): - return encode_sig(*ecdsa_raw_sign(electrum_sig_hash(msg), priv)) - -def ecdsa_raw_verify(msghash, vrs, pub): - v, r, s = vrs - - w = inv(s, N) - z = hash_to_int(msghash) - - u1, u2 = z * w % N, r * w % N - x, y = fast_add(fast_multiply(G, u1), fast_multiply(decode_pubkey(pub), u2)) - - return r == x - -def ecdsa_verify(msg, sig, pub): - return ecdsa_raw_verify(electrum_sig_hash(msg), decode_sig(sig), pub) - -def estimate_tx_size(ins, outs, txtype='p2pkh'): - '''Estimate transaction size. - Assuming p2pkh: - out: 8+1+3+2+20=34, in: 1+32+4+1+1+~73+1+1+33=147, - ver:4,seq:4, +2 (len in,out) - total ~= 34*len_out + 147*len_in + 10 (sig sizes vary slightly) - ''' - if txtype=='p2pkh': - return 10 + ins*147 +34*outs - else: - raise NotImplementedError("Non p2pkh transaction size estimation not"+ - "yet implemented") diff --git a/bitcoin/podle.py b/bitcoin/podle.py new file mode 100644 index 00000000..f80ab5d6 --- /dev/null +++ b/bitcoin/podle.py @@ -0,0 +1,635 @@ +#Proof Of Discrete Logarithm Equivalence +#For algorithm steps, see https://gist.github.com/AdamISZ/9cbba5e9408d23813ca8 +import secp256k1 +import os +import hashlib +import json +from py2specials import * +from py3specials import * +from secp256k1_main import ctx +PODLE_COMMIT_FILE = None +N = 115792089237316195423570985008687907852837564279074904382605163141518161494337 +dummy_pub = secp256k1.PublicKey(ctx=ctx) + +def set_commitment_file(file_loc): + global PODLE_COMMIT_FILE + PODLE_COMMIT_FILE = file_loc + +def get_commitment_file(): + return PODLE_COMMIT_FILE + +class PoDLEError(Exception): + pass + +class PoDLE(object): + """See the comment to PoDLE.generate_podle for the + mathematical structure. This class encapsulates the + input data, the commitment and the opening (the "proof"). + """ + + def __init__(self, u=None, priv=None, P= None, P2=None, s=None, e=None, + used=False): + #This class allows storing of utxo in format "txid:n" only for + #convenience of storage/access; it doesn't check or use the data. + #Arguments must be provided in hex. + self.u = u + if not priv: + if P: + #Construct a pubkey from raw hex + self.P = secp256k1.PublicKey(safe_from_hex(P), raw=True, ctx=ctx) + else: + self.P = None + else: + if P: + raise PoDLEError("Pubkey should not be provided with privkey") + #any other formatting abnormality will just throw in PrivateKey + if len(priv)==66 and priv[-2:]=='01': + priv = priv[:-2] + self.priv = secp256k1.PrivateKey(safe_from_hex(priv), ctx=ctx) + self.P = self.priv.pubkey + if P2: + self.P2 = secp256k1.PublicKey(safe_from_hex(P2), raw=True, ctx=ctx) + else: + self.P2 = None + #These sig values should be passed in hex. + if s: + self.s = safe_from_hex(s) + if e: + self.e = safe_from_hex(e) + #Optionally maintain usage state (boolean) + self.used = used + #the H(P2) value + self.commitment = None + + def mark_used(self): + self.used = True + + def mark_unused(self): + self.used = False + + def get_commitment(self): + """Set the commitment to sha256(serialization of public key P2) + Return in hex to calling function + """ + if not self.P2: + raise PoDLEError("Cannot construct commitment, no P2 available") + if not isinstance(self.P2, secp256k1.PublicKey): + raise PoDLEError("Cannot construct commitment, P2 is not a pubkey") + self.commitment = hashlib.sha256(self.P2.serialize()).digest() + return safe_hexlify(self.commitment) + + def generate_podle(self, index=0): + """Given a raw private key, in hex format, + construct a commitment sha256(P2), which is + the hash of the value x*J, where x is the private + key as a raw scalar, and J is a NUMS alternative + basepoint on the Elliptic Curve; we use J(i) where i + is an index, so as to be able to create multiple + commitments against the same privkey. The procedure + for generating the J(i) value is shown in getNUMS(). + Also construct a signature (s,e) of Schnorr type, + which will serve as a zero knowledge proof that the + private key of P2 is the same as the private key of P (=x*G). + Signature is constructed as: + s = k + x*e + where k is a standard 32 byte nonce and: + e = sha256(k*G || k*J || P || P2) + + Possibly Joinmarket specific comment: + Users *should* generate with lower indices first, + since verifiers will give preference to lower indices + (each verifier may have their own policy about how high + an index to allow, which really means how many reuses of utxos + to allow in Joinmarket). + + Returns a commitment of form H(P2) which, note, will depend + on the index choice. Repeated calls will reset the commitment + and the associated signature data that can be used to open + the commitment. + """ + #TODO nonce could be rfc6979? + k = os.urandom(32) + J = getNUMS(index) + KG = secp256k1.PrivateKey(k, ctx=ctx).pubkey + KJ = J.tweak_mul(k) + self.P2 = getP2(self.priv, J) + self.get_commitment() + self.e = hashlib.sha256(''.join( + [x.serialize() for x in [KG, KJ, self.P, self.P2]])).digest() + k_int = decode(k, 256) + priv_int = decode(self.priv.private_key, 256) + e_int = decode(self.e, 256) + sig_int = (k_int + priv_int*e_int) % N + self.s = encode(sig_int, 256, minlen=32) + return self.reveal() + + def reveal(self): + """Encapsulate all the data representing the proof + in a dict for client functions. Data output in hex. + """ + if not all([self.u, self.P, self.P2, self.s, self.e]): + raise PoDLEError("Cannot generate proof, data is missing") + if not self.commitment: + self.get_commitment() + Phex, P2hex, shex, ehex, commit = [ + safe_hexlify(x) for x in [self.P.serialize(), + self.P2.serialize(), + self.s, self.e, self.commitment]] + return {'used': str(self.used), 'utxo': self.u, 'P': Phex, 'P2': P2hex, + 'commit': commit, 'sig': shex, 'e': ehex} + + def serialize_revelation(self, separator='|'): + state_dict = self.reveal() + ser_list = [] + for k in ['utxo', 'P', 'P2', 'sig', 'e']: + ser_list += [state_dict[k]] + ser_string = separator.join(ser_list) + return ser_string + + @classmethod + def deserialize_revelation(cls, ser_rev, separator='|'): + ser_list = ser_rev.split(separator) + if len(ser_list) != 5: + raise PoDLEError("Failed to deserialize, wrong format") + utxo, P, P2, s, e = ser_list + return {'utxo':utxo, 'P': P, 'P2': P2, 'sig': s, 'e': e} + + def verify(self, commitment, index_range): + """For an object created without a private key, + check that the opened commitment verifies for at least + one NUMS point as defined by the range in index_range + """ + if not all([self.P, self.P2, self.s, self.e]): + raise PoDLE("Verify called without sufficient data") + if not self.get_commitment() == commitment: + return False + for J in [getNUMS(i) for i in index_range]: + sig_priv = secp256k1.PrivateKey(self.s, raw=True, ctx=ctx) + sG = sig_priv.pubkey + sJ = J.tweak_mul(self.s) + e_int = decode(self.e, 256) + minus_e = encode(-e_int % N, 256, minlen=32) + minus_e_P = self.P.tweak_mul(minus_e) + minus_e_P2 = self.P2.tweak_mul(minus_e) + KG = dummy_pub.combine([sG.public_key, minus_e_P.public_key]) + KJ = dummy_pub.combine([sJ.public_key, minus_e_P2.public_key]) + KGser = secp256k1.PublicKey(KG, ctx=ctx).serialize() + KJser = secp256k1.PublicKey(KJ, ctx=ctx).serialize() + #check 2: e =?= H(K_G || K_J || P || P2) + e_check = hashlib.sha256( + KGser + KJser + self.P.serialize() + self.P2.serialize()).digest() + if e_check == self.e: + return True + #commitment fails for any NUMS in the provided range + return False + +def getG(compressed=True): + """Returns the public key binary + representation of secp256k1 G + """ + priv = "\x00"*31 + "\x01" + G = secp256k1.PrivateKey(priv, ctx=ctx).pubkey.serialize(compressed) + return G + +def getNUMS(index=0): + """Taking secp256k1's G as a seed, + either in compressed or uncompressed form, + append "index" as a byte, and append a second byte "counter" + try to create a new NUMS base point from the sha256 of that + bytestring. Loop counter and alternate compressed/uncompressed + until finding a valid curve point. The first such point is + considered as "the" NUMS base point alternative for this index value. + + The search process is of course deterministic/repeatable, so + it's fine to just store a list of all the correct values for + each index, but for transparency left in code for initialization + by any user. + + The NUMS generator generated is returned as a secp256k1.PublicKey. + """ + + assert index in range(256) + nums_point = None + for G in [getG(True), getG(False)]: + seed = G + chr(index) + for counter in range(256): + seed_c = seed + chr(counter) + hashed_seed = hashlib.sha256(seed_c).digest() + #Every x-coord on the curve has two y-values, encoded + #in compressed form with 02/03 parity byte. We just + #choose the former. + claimed_point = "\x02" + hashed_seed + try: + nums_point = secp256k1.PublicKey(claimed_point, raw=True, ctx=ctx) + return nums_point + except: + continue + assert False, "It seems inconceivable, doesn't it?" # pragma: no cover + +def verify_all_NUMS(write=False): + """Check that the algorithm produces the expected NUMS + values; more a sanity check than anything since if the file + is modified, all of it could be; this function is mostly + for testing, but runs fast with pre-computed context so can + be run in user code too. + """ + nums_points = {} + for i in range(256): + nums_points[i] = safe_hexlify(getNUMS(i).serialize()) + if write: + with open("nums_basepoints.txt", "wb") as f: + from pprint import pformat + f.write(pformat(nums_points)) + assert nums_points == precomp_NUMS, "Precomputed NUMS points are not valid!" + + +def getP2(priv, nums_pt): + """Given a secp256k1.PrivateKey priv and a + secp256k1.PublicKey nums_pt, an alternate + generator point (note: it's in no sense a + pubkey, its privkey is unknowable - that's + just the most easy way to manipulate it in the + library), calculate priv*nums_pt + """ + priv_raw = priv.private_key + return nums_pt.tweak_mul(priv_raw) + +def get_podle_commitments(): + """Returns set of commitments used as a list: + [H(P2),..] (hex) and a dict of all existing external commitments. + It is presumed that each H(P2) can + be used only once (this may not literally be true, but represents + good joinmarket "citizenship"). + This is stored as part of the data in PODLE_COMMIT_FILE + Since takers request transactions serially there should be no + locking requirement here. Multiple simultaneous taker bots + would require extra attention. + """ + if not os.path.isfile(PODLE_COMMIT_FILE): + return ([], {}) + with open(PODLE_COMMIT_FILE, "rb") as f: + c = json.loads(f.read()) + if 'used' not in c.keys() or 'external' not in c.keys(): + raise PoDLEError("Incorrectly formatted file: " + PODLE_COMMIT_FILE) + return (c['used'], c['external']) + +def add_external_commitments(ecs): + """To allow external functions to add + PoDLE commitments that were calculated elsewhere; + the format of each entry in ecs must be: + {txid:N:{'P':pubkey, 'reveal':{1:{'P2':P2,'s':s,'e':e}, 2:{..},..}}} + """ + update_commitments(external_to_add=ecs) + +def update_commitments(commitment=None, external_to_remove=None, + external_to_add=None): + """Optionally add the commitment commitment to the list of 'used', + and optionally remove the available external commitment + whose key value is the utxo in external_to_remove, + persist updated entries to disk. + """ + c = {} + if os.path.isfile(PODLE_COMMIT_FILE): + with open(PODLE_COMMIT_FILE, "rb") as f: + try: + c = json.loads(f.read()) + except ValueError: + print "the file: " + PODLE_COMMIT_FILE + " is not valid json." + sys.exit(0) + + if 'used' in c: + commitments = c['used'] + else: + commitments = [] + if 'external' in c: + external = c['external'] + else: + external = {} + if commitment: + commitments.append(commitment) + #remove repeats + commitments = list(set(commitments)) + if external_to_remove: + external = { + k: v for k, v in external.items() if k not in external_to_remove} + if external_to_add: + external.update(external_to_add) + to_write = {} + to_write['used'] = commitments + to_write['external'] = external + with open(PODLE_COMMIT_FILE, "wb") as f: + f.write(json.dumps(to_write, indent=4)) + +def generate_podle(priv_utxo_pairs, tries=1, allow_external=None): + """Given a list of privkeys, try to generate a + PoDLE which is not yet used more than tries times. + This effectively means satisfying two criteria: + (1) the generated commitment is not in the list of used + commitments + (2) the index required to generate is not greater than 'tries'. + Note that each retry means using a different generator + (see notes in PoDLE.generate_podle) + Once used, add the commitment to the list of used. + If we fail to find an unused commitment with this algorithm, + we fallback to sourcing an unused commitment from the "external" + section of the commitments file; if we succeed in finding an unused + one there, use it and add it to the list of used commitments. + If still nothing available, return None. + """ + used_commitments, external_commitments = get_podle_commitments() + for priv, utxo in priv_utxo_pairs: + for i in range(tries): + #Note that we will return the *lowest* index + #which is still available. + p = PoDLE(u=utxo, priv=priv) + c = p.generate_podle(i) + if c['commit'] in used_commitments: + continue + #persist for future checks + update_commitments(commitment=c['commit']) + return c + if allow_external: + filtered_external = dict( + [(x, external_commitments[x]) for x in allow_external]) + for u, ec in filtered_external.iteritems(): + #use as many as were provided in the file, up to a max of tries + m = min([len(ec['reveal'].keys()), tries]) + for i in [str(x) for x in range(m)]: + p = PoDLE(u=u,P=ec['P'],P2=ec['reveal'][i]['P2'], + s=ec['reveal'][i]['s'], e=ec['reveal'][i]['e']) + if p.get_commitment() not in used_commitments: + update_commitments(commitment=p.get_commitment()) + return p.reveal() + #If none of the entries in the 'reveal' list for this external + #commitment were available, they've all been used up, so + #remove this entry + if m == len(ec['reveal'].keys()): + update_commitments(external_to_remove=u) + #Failed to find any non-used valid commitment: + return None + +def verify_podle(Pser, P2ser, sig, e, commitment, index_range = range(10)): + verifying_podle = PoDLE(P=Pser, P2=P2ser,s=sig,e=e) + #check 1: Hash(P2ser) =?= commitment + if not verifying_podle.verify(commitment, index_range): + return False + return True + +precomp_NUMS = {0: + '0296f47ec8e6d6a9c3379c2ce983a6752bcfa88d46f2a6ffe0dd12c9ae76d01a1f', + 1: '023f9976b86d3f1426638da600348d96dc1f1eb0bd5614cc50db9e9a067c0464a2', + 2: '023745b000f6db094a794d9ee08637d714393cd009f86087438ac3804e929bfe89', + 3: '023346660dcb1f8d56e44d23f93c3ad79761cdd5f4972a638e9e15517832f6a165', + 4: '02ec91c86964dcbb077c8193156f3cfa91476d5adfcfcf64913a4b082c75d5bca7', + 5: '02bbc5c4393395a38446e2bd4d638b7bfd864afb5ffaf4bed4caf797df0e657434', + 6: '02967efd39dc59e6f060bf3bd0080e8ecf4a22b9d1754924572b3e51ce2cde2096', + 7: '02cfce8a7f9b8a1735c4d827cd84e3f2a444de1d1f7ed419d23c88d72de341357f', + 8: '0206d6d6b1d88936bb6013ae835716f554d864954ea336e3e0141fefb2175b82f9', + 9: '021b739f21b981c2dcbaf9af4d89223a282939a92aee079e94a46c273759e5b42e', + 10: '025d72106845e03c3747f1416e539c5aa0712d858e7762807fdc4f3757fd980631', + 11: '02e7d4defb5d287734a0f96c2b390aa14f5f38e80c5a5e592e4ce10d55a5f5246b', + 12: '023c1bf301bcfa0f097f1a3931c68b4fd39b77a28cc7b61b2b1e0b7ca6d332493c', + 13: '0283ac2cdd6b362c90665802c264ee8e6342318070943717faee62ef9addeff3e9', + 14: '02cb9f6164cd2acdf071caef9deab870fc3d390a09b37ba7af8e91139b817ce807', + 15: '02f0a3a3e22c5b04b6fe97430d68f33861c3e9be412220dc2a24485ea5d55d94db', + 16: '02860ca3475757d90d999e6553e62c07fce5a6598d060cceeead08c8689b928095', + 17: '0246c8eabc38ce6a93868369d5900d84f36b2407eecb81286a25eb22684355b41d', + 18: '026aa6379d74e6cd6c721aef82a34341d1d15f0c96600566ad3fa8e9c43cbb5505', + 19: '02fdeacb3b4d15e0aae1a1d257b4861bcc9addb5dc3780a13eb982eb656f73d741', + 20: '021a83ecfaeb2c057f66a6b0d4a42bff3fe5fda11fe2eea9734f45f255444cddc0', + 21: '02d93580f3e0c2ec8ea461492415cc6a4be00c50969e2c32a2135e7d04f112309a', + 22: '0292c57be6c3e6ba8b44cf5e619529cf75e9c6b795ddecd383fb78f9059812cb3f', + 23: '02480f099771d0034d657f6b00cd17c7315b033b19bed9ca95897bc8189928dd47', + 24: '02ac0701cdc6f96c63752c01dc8400eab19431dfa15f85a7314b1e9a3df69a4a66', + 25: '026a304ceb69e37d655c1ef100d7ad23192867151983ab0d168af96afe7f1997f6', + 26: '023b9ff8e4a853b29ecae1e8312fae53863e86b8f8cb3155f31f7325ffb2baf02c', + 27: '021894ce66d61c33e439f38a36d92c0e45bf28dbc7e30bfb4d7135b87fc8e890e1', + 28: '02d9e7680e583cf904774d4c19f36cb3d238b6c770e1e7db03f444dc8b15b29687', + 29: '024350c7ff5b2bf2c58e3b17a792716d0e76cff7ad537375d1abc6e249466b25a3', + 30: '02c6577e1cdcbcfadb0ae037d01fbf6d74786eecdb9d1ee277d9ba69b969728cfe', + 31: '029f395b4c7b20bcb6120b57bee6d2f7353cd0aa9fe246176064068c1bd9b714d1', + 32: '02d180786087720b827bf04ae800547102470a1e43de314203e90228c586b481a1', + 33: '023548173a673965c18d994028bc6d5f5df1f60dccf9368b0eae34f8cff3106943', + 34: '02118124c53b86fdade932c4304ad347a19ce0af79a9ab885d7d3a6358a396e360', + 35: '02930bcdee5887fa5a258335d6948017e6d7f2665b32dcc76a84d5ca7cd604d89b', + 36: '0267e79a47058758a8ee240afd941e0ae8b4f175f29a3cf195ad6ff0e6d02955b1', + 37: '027e53d9fb04f1bb69324245306d26aa60172fd13d8fe27809b093222226914de6', + 38: '02ef09fbdcd22e1be4f0d4b2d13a141051b18009d7001f6828c6a40b145c9df23e', + 39: '028742fd08c60ba13e78913581db19af2f708c7ec53364589f6cbcf9d1c8b5105f', + 40: '020ce14308d2f516bf4f9e0944fb104907adef8f4c319bfcc3afab73e874a9ce4a', + 41: '027635f125f05a2548201f74c4bbdcbe89561204117bd8b82dfae29c85a576a58e', + 42: '02fe878f3ae59747ee8e9c34876b86851d5396124e1411f86fe5c58f08f413a549', + 43: '02f2a6af33bd08ab41a010d785694e9682fa1cc65733f30a53c40541d1c1bfb660', + 44: '02cbe9d18b6d5fc9993ef862892e5b2b1ea5d2710a4f208672c0f7c36a08bb5686', + 45: '023fb079b25c0a8241465fb55802f22ebb354e6da81f7dabfe214ddbd9d3dfcd5a', + 46: '021a5b234b9a10fc5f08ed9c1a136a250e92156adc12109a97dd7467276d6848a8', + 47: '0240fbe9363d50585da40aef95f311fc2795550e787f62421cd9b6e2f719bb9547', + 48: '02a245fbbc00f1d6feb72a9e1d3fd0033522839d33440aea64f52e8bccee616be8', + 49: '02fd1e94bb23a4306de64064841165e3db497ae5b246dabff738eb3e6ea51685a7', + 50: '0298362705914c839e45505369e54faefbb3aaebb4c486b4d6e59ca03304f3552c', + 51: '021b8109a23b858114d287273620dd920029d84b90f63af273c1c78492b1a70105', + 52: '028df6ce4fec30229cddb86c62606cff80e95cb8277028277f3dcc8ac9f98eef9d', + 53: '02ed02925d806df4ac764769d11743093708808157fb2933eb19af5399dcfd500c', + 54: '02ce88da0e81988bd8f5d63ad06898a355f7dc7f46bb08cf5f1e9bc5c3752ad13c', + 55: '02f4868cc8285cd8d74d4213d18d53d5f410d50223818f1be6fe8090904e03743d', + 56: '02770cecdf18aa2115b6e5c4295468f2e92a53068dc4295d0e5d0890b71d1a2fcc', + 57: '02b5d4dce8932de37c6ef13a7f063f164dfd07f7399e8e815e22b5af420608fd2a', + 58: '0284ad07924dbac50a72455aec3ddba50b1ed71e678ba935bb5d95c8a8232b1353', + 59: '02cb8c916a6f9bc39c8825f5b0378bb1b0a0679e191843aa4db2195b81f14c87e0', + 60: '0235aa30ec3df8dd193a132dbaf3b351af879c59504ed8b7b5ad5f1f1ea712854f', + 61: '02df91206e955cefe7bcda4555fc6ad761b0e98d464629f098d4483306851704e9', + 62: '02ed4f1fccd47e66a8d74e58b4f6e31b5172b628fc0dacdb408128c914eb80f506', + 63: '0263991bb62aaca78a128917f5c4e15183f98aefddf04070c5ca537186f1c1a97a', + 64: '02ffe2b017882d57db27864446ad7b21d3855ae64bddf74d46e3a611bf903580be', + 65: '02d647aba2c01eecd0fac7e82580dd8b92d66db7341d1b65a5e4b01234f1fbb2cd', + 66: '023134ff85401dba9aff426d3f3ba292ea59684b8c48ea0b495660797a839246a6', + 67: '02827880fe0410c9ea84f75a629f8f8e6eed1f26528af421cf23b8ecf93b6b4b7b', + 68: '02859b3f9f1f5ba6aa0787f8d3f3f2f21b4932c40bc36b6669383e3bbd19654a5f', + 69: '02a7d204dfc3eed44abd0419202e280f6772efd5acf9fd34331b8f47c81c6dab19', + 70: '02e15d11b443a9340ac31a8c5774ce34cd347834470c8d68c959828fae3a7eb0c6', + 71: '029931f65e46627d60519bfd08bd8a1bb3d8d2921f7f8c9ef31f4bfcdd8028ead2', + 72: '02e5415ba78743d736018f19757ee0e1ca5d4a4fb1d0464cd3eea8d89b34dd37b8', + 73: '027ea7860afc3de502d056d9a19ca330f16cd61cfefbeb768df68a882d1f8f15f5', + 74: '026c19becac43626582622e2b7e86ebd8056f40aa8ab031e70f4deae8cab34503f', + 75: '02098dab044c888ddebe6713fcb8481f178e3ba42d63310b08d8234e20fe1de13f', + 76: '02ed6af1a2bebcb381ce92f87638267b1afefe7a1cdce16253f5bf9f99a84ce4b2', + 77: '023d8493f9e72cd3212166de50940d980f603ae429309abb29e15cccc1983efe37', + 78: '025c07d7513b1bae52a0089a4faee127148e2ba5a651983083aedc1ae8403cf1eb', + 79: '0285a93a8c8e6134b3a53c5bd1b5b7d24e7911763ea887847c5d66af172ed17f10', + 80: '02fea28fb142aa95fcd44398c9482a3c185ec22fee8f24ad6b2297ac7478423f21', + 81: '02f9840a1635ae3fa131405526974d40d2edee17adf58278956373ce6c69757c2a', + 82: '023579e441a7dcdbd36a2932c64fa3318023b1f3d04daab148622b7646246a6d7c', + 83: '02bcbc2933f90a88996c1363c8d3a7004e0c6b75040041201fb022e45acb0af6a7', + 84: '02cd52e0d28f5564fc2bf842fa63dfefbcf2bb5fe0325703c132be5cd14cca7291', + 85: '021e648e261b93fedd3439352899c0fa1acedd1f68ab508050a13ed3cbbc93c2ff', + 86: '0295f9caea5f57d11b12ddee154a36a14921a8980fa76726e48e1d76443d4e306f', + 87: '02396edf4c18283dd3ef68a2c57b642bd87ae9f8b6be5e5fe4a41c5b86c5db8eb2', + 88: '0264f323ca3eee79385c9bfd35cd4cf576e51722f38dd98531d531a29913e5170d', + 89: '02facd3f63f543e0ab9b13323340113acbe8ed3bafdfabdc80626cdd15386c80f3', + 90: '02b6762640f96367fbf65eecfafcee5c6f7d6a42b706113053bb36a882659d3e65', + 91: '02ed63f2eca15d9b338fcdb9b3efa3b326e173a1390706258829680f7973fa851c', + 92: '026f6d47d0d48ff13d64ec6a1db2dc51173cee86ab8010a17809b3fe01483d9fc5', + 93: '02814e7cae580a1ef86d6ee9b2f9f26fe771e8ea47acf11153b04680ada9cd3042', + 94: '020e46225fb3ee8f04d08ffbe12d9092ff7f7227f9cb55709890c669e8a1c97963', + 95: '028194469e8d6ee660e95d6125ba0152ad5c24bf7e452adf80db7062d6926851c4', + 96: '02b3e1f5754562635ebeecfd32edb0d84a79b2f0c270bac153e60dd29334dc2663', + 97: '02afff20730724a2d422f330e962362e7831753545ac0a931dd94be011ccf93e9c', + 98: '02a9cfdf0471a34babfc2f6201dbc79530f3f319204daedb7ec05effc2bdfc5a74', + 99: '02838fe450f2dd0c460b5fae90ec2feb5b7f001f9cd14c01a475c492cf16ea594b', + 100: '02aacc3145d04972d0527c4458629d328219feda92bef6ef6025878e3a252e105a', + 101: '02720fe09616d4325d3c4c702a0aeafbbbff95ef962af531c5ae9461ec81fdf8c5', + 102: '02e6408f24461a6c484f6c4493c992d303211d5e4297d34afede719a2b70c96c14', + 103: '02b9ecf2d3fdf2611c6d4be441a0f9a3810dadae39feb3c0d855748cc2dd98a968', + 104: '027a32d12a536af038631890a9b90ee20b219c9c8231a95b1cde24c143d8173fec', + 105: '02d26c98fb50b57b7defdf1e8062a52b2a859ba42f3d1760ee8ff99c4e9eb3ec03', + 106: '02df85556e8d1e97a8093e4d9950905ebced0ea9a1e49728713df1974eeb455774', + 107: '021fe1dbada397155a80225b59b4fb9a32450a991b2d9d11d8500e98344927c856', + 108: '0211ccd0980a9ab6f4bb82fdc2e2d1ddace063a7bc1914a6ab4d02b0fa1ca746ec', + 109: '0264bd41f41aad19f8bfd290fd3af346ebbf80efd33f515854f82bd57e9740f7aa', + 110: '0226d5fb607cadb8720e900ce9afb9607386ad7b767e4ab3a4e0966223324b92eb', + 111: '02b3bbf2e2ceae25701bd3b78ba13bea3f0dfed7581b8a8a67c66de9fd96ee41e2', + 112: '024b8dd765e385d0e04772f3dbf1b1a82abc2de3e5740baac1f6306cd9fd45fe99', + 113: '022153f6a884ae893ebb0642a84d624c0b62894d7cb9e2a48a3a0c4696e593f9db', + 114: '0245e22b6388cb14c9c8dbcac94853bdf1e81816c07e926a82b96fc958aa874626', + 115: '02cba97826b089c695b1acffdcdbf1484beec5eb95853fea1535d6d7bdb4e678b0', + 116: '02ed006fbab2d18adbd96d2f1de6b83948e2a47acc8d2f92d7af9ba01ffae58276', + 117: '02513592f4434ee62802d3965f847684693000830107c72cd8de4b34e05b532dae', + 118: '028adc75647453a247bd44855abb56b60794aaed5ce21c9898e62adac7adcfbe8e', + 119: '02a712d5dc572086359f1688e8e7b9a5f7fc3079644aea27cdddb382208fee885b', + 120: '029abf8551218c9076f6d344baa099041fe73e5e844aac6e5c20240834105cdf60', + 121: '027d480071a2d128c51e84c380467e1ac8435f05b985bbfee0099d35b4121fb0ca', + 122: '02a7f2e4253fa0d833beca742e210c0d59a4ffc8559764766dcffb1aa3e4961826', + 123: '023521309a6bdfafdf7bdae574a5f6010eb992e4bae46d8f83c478eac137889270', + 124: '02b99fe8623aa19ca2bed6fe435ae95c5072a40193913bebe5466f675c92a31db7', + 125: '02dc035112a2b4881917ea1db159e7f35ee9d98d31533e1285ca150ce84e538e4f', + 126: '0291a07ecce8061561624de7348135b9081c5edd61541b24fa002fb6c074318fec', + 127: '020d8a5253d7e0166aa37680a5f64cab0cdad2cdc4c0e8ae61d310df4c4f7386eb', + 128: '026285db47fee60b5ad54cbd4c27a4e0cd723b86a920f03b12dc9b8c5f19f06448', + 129: '020f94a9df4302f701b4629f74d401484daf84c7aabaf533f8c21c1626009e923c', + 130: '027bb78af54b01ddad4e96b51a4e024105b373aab7e1a6ec16279967fcbbb096b4', + 131: '02e1b20c0da3b8c991f8909fd0d31874be00e9fcb130d7c28b8ad53326cdf13755', + 132: '02bbdd4dfc047f216e2cbff789bcf850423bedf2006d959963f75621810fecf0d9', + 133: '024e1fe4b23feda8651a467090e0ce7e8b8db2ccb1c27d52255c76754aa1940d1b', + 134: '0241aad8f575556c49c4fefae178c2c38541962bfff2ca84ebecea9f661ccf3536', + 135: '02bcf6203d725ca0640bd045389e854e00087c54ba01fd739c6ef685b22f89340c', + 136: '0202178e6b3a9b498399aa392b32dc9010f1eea322a6d439ad0c8cacf2008b3e34', + 137: '026db3289d470df0fdf04f5f608fae2d7ec4ddbd3de2603f6685789520bdee01fc', + 138: '0239bcfc796488129e3b2f01e6fbbda2f1b357b602e94b5091b44c916e9806dc34', + 139: '020513bc4a618d32d784083f13d46e6c6d547f01b24942351760f6dc42e2bb7167', + 140: '0204d2495e4fc20e0571ab2fcb4c1989fdda4542923aa97fe1a77a11c79ade1964', + 141: '021eaa6af99ea4f1143a45a1b5af7b2d3c3e8810f358be6261248c5ba2492a7b4e', + 142: '02799849e87e3862170add5b28a3b7e54b04cc60c2cec39de7eca9bfdfaaf930a8', + 143: '02639bced287084268136c5b6e9e22f743b6c8f813e6aabe39521715bfa4a46ab8', + 144: '0283c8b21fc038c1fbeedfae0b3abc4dbde672b0dcfda540f9fcfcf8c6e6d29fc3', + 145: '02b284f4510535ff98e683f25c08b7ae7dd19f7b861e70a202469ddfb2877bc729', + 146: '0256af1c82cde40ffd03564368b8256a5e48ef056df2655013f0b1aa15de1de8d2', + 147: '02964b55eab2f19518ee735cae2f7f780bfab480bcbd360f7a90a2904301203366', + 148: '02f046486f4a473f2226f6bd120aafc55a5c8651f3eb0855aa6a821f69f3016cc6', + 149: '02eb8dfb7c59fbf24671e258ca5e8eda3ea74c5f0455eed4987cfda79f4fcf823f', + 150: '020fac2c37cc273d982c07b2719a3694348629d5bdaebc22967fb9d0e1d7f01842', + 151: '025c0c8ff9a102f99f700081526d2b93b9d51caf81dcf4d02e93cf83b4a7ff5c92', + 152: '02a118f5fa9c5ef02707e021f9cb8056e69018ef145bec80ead4e09c06a60050c1', + 153: '029ea72333d1908bb082bffec9da8824883df76a89709ab090df86c45be4abf784', + 154: '02bacc52256e5221dbfc9a3f22e30fa8e86ddd38e3877e3dc41de91bdcf989b00b', + 155: '02bc8b37dc66e2296ae706c896f5b86bd335f724cfa9783e41b9dc5e901b42b1de', + 156: '02eca1099cea9bcab80820d6b64aec16dce1efa0e997b589f6dba3a8fd391fb100', + 157: '027f1c1bb99bd1a0e486f415f8960d45614a6fcac8cedc260e07197733844827d0', + 158: '021fc54df458bcfafc8a83d4759224c49c4b338cf23cd9825d6e9cdeffc276375b', + 159: '027d4fff88da831999ba9b21e19baf747dc26ea76651146e463d4f3e51c586ee91', + 160: '02e49c0fef0ebc52908cdcea4d913a42e5f24439fffdfaa21cc55a6add0ad9d122', + 161: '0208b5e8e5035fdb62517d4ebab0696775dbfbdba8ff80f2031c1156cda195a2ab', + 162: '0202e990bab267fff1575d6acc76fe2041f4196f4b17678872f9c160d930e5be35', + 163: '02c73fcedd9f6eabc8fe4e1e7211cdb0f28967391d200147d46e4077d2915c262d', + 164: '0261490abc5f14387ef585f42d99dbddb0837b166694d4af521086a1ffd46e5640', + 165: '02b46a143e4e0af20a12c39f3105aca57ca79f9332df67619ee859b5d9bffb6d6d', + 166: '0299f53c064d068f003f8871acae31b84ddda9d8dbe516d02dc170c70314ee2af7', + 167: '023305144dccba65c67001474ee1135aa96432f386b5eb27582393b2ed4bfc185d', + 168: '02e044b70ff7e9c784b3c40d09bdfadd4a037e692b0b3aa9ab6bb91203f86a0b37', + 169: '02ded067a2e44282b0d731a28ffbd03ca6046c5b1a262887ea7cab4810050fbb8c', + 170: '02e00e4c9198194d92a93059bce61f8249e1006eee287aa94fe51bb207462e5492', + 171: '0241b89d9164f4c07595ca99b7d73cad2b20ac39847cf703dff1d7d6add339ebeb', + 172: '02eba24cd4946e149025a9bf7759df5362245bf7c53c5a3205be0c92c59db8d5dc', + 173: '026bd40c611246a789521c46d758a80337ff40bb298a964612b2af74039211727a', + 174: '02b9095e071e4edfddf8afb0e176536957509d23f90fb7175ad086b4098e731c73', + 175: '0214ad0014dfddc5c7eb0801b97268c1b7e03d64215d6b9d5ed80b468089e4a01d', + 176: '02c455b8e38103ade8794fb51a1656e1439b42bdf79afd17a9df8542153914a7cf', + 177: '02cc89d6437fdcf711a76eb16f4014f2e21b71740afc8b3ec13ccb60a45b12d815', + 178: '0208eee5857dda0ae1c721e6ed4c74044add4e1ce66f105413e9ef1cccbdca87ad', + 179: '02edc663693827cad44d004ac24753bfc3167f81ff4074bb862453376593229c0f', + 180: '0202a4b7fb31e30b6d8f90a5442ef31f800902ea7a9511e24437b7a0ef516f79a9', + 181: '02ff05472c2019ac2c9ab8b7fcb0604a94b7379c350306be262144588ea252d0f4', + 182: '02b131bb594a1270d231e18459e484c49f3eca3b3b2291c9be81c01dc8a4037fa1', + 183: '02f50125277ea19f633e93868cf8e8a4cd76b21eedf8e3ef59de43f40d73a01d01', + 184: '027aab228a7d6f87003b01fb9c0b9bcfb2098adbc76f5f9b856aedd28077fc4471', + 185: '02925200e4f74bea719a99f4a0b05165b9af475f2187381bd0b79cad4d5f2593b6', + 186: '02c311f1750c6d5c364b71c3b0f369f6959d34a3718da695c5b227ecf1a4669bf6', + 187: '02cb030c71169d0a1ae30ffba92311bc06bb64b27570598dedabdea0b24631a0ca', + 188: '02e64669898eecff7aa887307be696a694f61559e7ca41119677b7e94f37cd2914', + 189: '028fe93e32c24df7f8aaf8d777335fd9ce9f9b5c121dec2ab1ff21575c047497e7', + 190: '026f08c1c3cb4cff5cdbd7985db4a8ebf0ebc0924530b0fa118d095c4667efeb52', + 191: '02afe08dbba6c999efb73aeae1da0ad8b143a1b51759caffd3ed2de4494adc47fb', + 192: '02e99aec0b5e869b3885a3b9f527fd3c546dde83d41a5a156703d0da5e10e04743', + 193: '02b7e5f4cb9233107bf7a47789dca4eb811af108822f2d4bd03dec13251ec45984', + 194: '023b971e135daa0b851797b17e3a1cc5ac8a9a6207a2e784a0fe36732a00407b49', + 195: '02b1742739bfbb528b2a2731cb5d5f1bd03f4fa9c94607837e586c7c6f6589be4a', + 196: '022cd1b023bb2afc68ee27b40f8deb1d1c6d7b7aa97c32c444f1ceebd449dbeb22', + 197: '02704e21f8bf38158d7e8100e297adfc930c14c8791beee9b907407f4ca654d95b', + 198: '02caabeb678374ca75bd815c370b2e37fb0470591557219d6289b1b1e655ed80c6', + 199: '026aa8d45112aa0da335054194c739e04787526250493f5a0eaaa8a346541d1a0f', + 200: '022fb12408355439bbee33066bbeefcffb0bdc9cfd1950510fd2a42bdc4eaa1d53', + 201: '02639fe47769f7694ca6dbfd934762472391d70b23868a58e11d2bd46373e1df29', + 202: '02f75360f52df674247c5f005b3451ee47becf3204862154d4e7ee97a0e40df3d2', + 203: '0230241e27d0d3ad727d26472541fcd48f2bb128db5611237fa9f33f86ede8d5c9', + 204: '0255d5a0aa37a226c001f6b7f19e2bddb10aeaa0652430b8defe35c3f03dfb3c0e', + 205: '024e6faa398b0acf8a8dfdd9d21e0a46a22d07cd0fcffd89749f74f94f9993f4d9', + 206: '020c1a256587306f58f274cc2238f651bbfadfd42436e6eb8f318ac08fae04e7ae', + 207: '025858b8188da173e8b01b8713b154ffae8b2d2eb8f9670362877102cf0c0c4f28', + 208: '02dc7509c77d7fa61c08c5525fb151bf4fe12deb1989a3be560a63105dae2ecd2e', + 209: '02a272df6dab1c22c209b45b601737c0077acb7869bb9fe264c991b4ef199e337d', + 210: '025168f2fdd730b4c33b57d3956e6a40dd27a4f32db70d9f9b5898fa2bed3de342', + 211: '028133baac70bc2c2ebe8a22af04b5faedd070e276c90e2f910bb9bf89441a80db', + 212: '029064628ebd6e97a945c1d52641a27bff3c4f59659e657b88d23c2ce1c4d04644', + 213: '023cf20c4e8675bce999a0128602fe21699db651540f3dcbe7a4ef2126243ba17a', + 214: '02cc685739a4b20e2d52ddf256e597c06b7eb69e65d009820c6744b739c7215340', + 215: '02d061544ce21398af3e0e6c329ce49976a9ecd804ebc543f4c16f6a32798f37c2', + 216: '029fe49ff440f23c69360a92d249db429bdc3601fc8a5a3fc1aa894de817c05490', + 217: '0222c8c4e90585f9816b5801bad43fb608857269fdaaefbe2b5b85903231685679', + 218: '0296b72ed4968860b733fb99846698df2e95c65af281b3ef8b5ab90e2d5de966cb', + 219: '02c27565a7fd5d1f4bcbe969bddbace99553fb65cb7750965350ff230b1f09f97d', + 220: '02e1254be9833236609bf44c62ef6da7188a44bbe2d53a72cf39a38ef9f99bb783', + 221: '0280663ce16afadc77e00ade780da53e7c11b02a66cbf36837ef7d9d2488f23417', + 222: '02ad8b11e62c6753917307bdde89a42896e0070d33f6f93c608d82f6d041b814a4', + 223: '02ce1d943dfc14654266507def2b7b9940bffceb4f54d709a149f99962083398fc', + 224: '023ea7eb26248c05beb4e4d8ba9f9785d5fd1a55d3137c90f40b807b60aa4262df', + 225: '0211c802fec9b31710d3849e2c1700cea5374ae422e54551946d96fc240c63fba0', + 226: '02204ad97ebe2ec30d6db1bfc1e1d4660331909668634c3cd928b5c369a6013367', + 227: '020251bf4271d359a082cdad23d9a5cd48916d78eed010fe1e7d9711cd420b3cdf', + 228: '0292b9757195350676e447e49425f887d3df7e27774bb3e0aab5b528da0a1a0340', + 229: '022be18362b2a167199a76f6065358063b1167d5bbcfe7652fc55f93a5ebd42e89', + 230: '02e6b1e618efe5f468bdb40f5ec167ed4fa7636849c4ff4ddab0199c903b37306c', + 231: '02a6676873de91890ecae000c575e46e4a9629865fb1662606da5e9c1fdcd55d5c', + 232: '02c088a3c96b13413caa5f32a8f4640e76ec0a37990577d679d2062e859547f058', + 233: '023e9703ed6209d5a25e0ecb34e04c22f274f37845aa2a4e2f2343e39928360e25', + 234: '02977d845787c4690152827bfd15e801044c84d33430a7ed928499e828cf131d14', + 235: '0224ea648555445d1305aaf6bd74fda3041b2a10bf7900a4c067462b01c6dc25f1', + 236: '02dfd472c98ece1dc2a18c1bebf98a09990fba673e725c029928937247022b9d24', + 237: '02a2a03933d06617adcf0f4ad692e95d463a5fa9938e8d451e5d6271f4a5af8bb4', + 238: '02ca24fa8d7aa53f7f5b4e1ca16eb6fd9b9cfb0162a332abb7a88ddf8e964c99bc', + 239: '02bbce92d1db3ef0c9c09793b760fd3b929c9168e4dff396c618fa0ed3cf6a5edb', + 240: '028af15d26d3b297f4d2aeaf308632b60251accf87aa8470b3d4d1ef2dabb99209', + 241: '021b81c0e878389231339fd9d622a736fc9d36de93a58ea6a4bc38fef86672278a', + 242: '021adc24309f605c7a5af106e8b930feaec0bec6545fb4c70b83ebe5cf341cab2d', + 243: '020462a3ff101ac379f87f43190459b7494f4128ea30035877ce22a35afb995e34', + 244: '02f1019851779a6d0db09e8abeba3b9a07b6931b43b0d973cfe261a96b4516cca4', + 245: '02d7023276f01ff22a9efeadd5b539d1d9ceb80ebf6813e6042a49c946a82f366f', + 246: '021594f45af3a21e0210a2ca4cbc3e95ea95db5aca3561fc1f759cb7f104dd0f62', + 247: '021398309b6c293c0dc28cdd7e55ad06306b59cb9c10d947df565e4a90f095a62a', + 248: '029f39d84383200e841187c5b0564e3b01a2ba019b86221c0c1dd3eae1b4dabb26', + 249: '0252ec719852f71c2d58886dd6ace6461a64677a368b7b8e220da005ac977abdc8', + 250: '0237f0d7de84b2cc6d2109b7241c3d49479066a09d1412c7a4734192715b021e06', + 251: '021e9e0e4784d15a29721c9a33fbcfb0af305d559c98a38dcf0ce647edd2c50caa', + 252: '02e705994a78f7942726209947d62d64edd062acfa8a708c21ac65de71e7ae71df', + 253: '0295f1cafd97e026341af3670ef750de4c44c82e6882f65908ec167d93d7056806', + 254: '023a0d381598e185bbff88494dc54e0a083d3b9ce9c8c4b86b5a4c9d5f949b1828', + 255: '02a0a8694820c794852110e5939a2c03f8482f81ed57396042c6b34557f6eb430a'} + diff --git a/bitcoin/secp256k1_main.py b/bitcoin/secp256k1_main.py index 660ed196..441bdbfd 100644 --- a/bitcoin/secp256k1_main.py +++ b/bitcoin/secp256k1_main.py @@ -12,8 +12,41 @@ import hmac import secp256k1 +#Global context for secp256k1 operations (helps with performance) ctx = secp256k1.lib.secp256k1_context_create(secp256k1.ALL_FLAGS) +#Standard prefix for Bitcoin message signing. +BITCOIN_MESSAGE_MAGIC = '\x18' + 'Bitcoin Signed Message:\n' + +"""A custom nonce function acting as a pass-through. +Only used for reusable donation pubkeys (stealth). +""" +from cffi import FFI + +ffi = FFI() +ffi.cdef('static int nonce_function_rand(unsigned char *nonce32,' + 'const unsigned char *msg32,const unsigned char *key32,' + 'const unsigned char *algo16,void *data,unsigned int attempt);') + +ffi.set_source("_noncefunc", +""" +static int nonce_function_rand(unsigned char *nonce32, +const unsigned char *msg32, +const unsigned char *key32, +const unsigned char *algo16, +void *data, +unsigned int attempt) +{ +memcpy(nonce32,data,32); +return 1; +} +""") + +ffi.compile() + +import _noncefunc +from _noncefunc import ffi + def privkey_to_address(priv, from_hex=True, magicbyte=0): return pubkey_to_address(privkey_to_pubkey(priv, from_hex), magicbyte) @@ -54,9 +87,11 @@ def num_to_var_int(x): elif x < 4294967296: return from_int_to_byte(254) + encode(x, 256, 4)[::-1] else: return from_int_to_byte(255) + encode(x, 256, 8)[::-1] -# WTF, Electrum? -def electrum_sig_hash(message): - padded = b"\x18Bitcoin Signed Message:\n" + num_to_var_int(len( +def message_sig_hash(message): + """Used for construction of signatures of + messages, intended to be compatible with Bitcoin Core. + """ + padded = BITCOIN_MESSAGE_MAGIC + num_to_var_int(len( message)) + from_string_to_bytes(message) return bin_dbl_sha256(padded) @@ -115,100 +150,25 @@ def from_wif_privkey(wif_priv, compressed=True, vbyte=0): return safe_hexlify(bin_key) def ecdsa_sign(msg, priv, usehex=True): - #Compatibility issue: old bots will be confused - #by different msg hashing algo; need to keep electrum_sig_hash, temporarily. - hashed_msg = electrum_sig_hash(msg) + hashed_msg = message_sig_hash(msg) if usehex: #arguments to raw sign must be consistently hex or bin hashed_msg = binascii.hexlify(hashed_msg) - dersig = ecdsa_raw_sign(hashed_msg, priv, usehex, rawmsg=True) - #see comments to legacy* functions - #also, note those functions only handles binary, not hex + sig = ecdsa_raw_sign(hashed_msg, priv, usehex, rawmsg=True) + #note those functions only handles binary, not hex if usehex: - dersig = binascii.unhexlify(dersig) - sig = legacy_ecdsa_sign_convert(dersig) + sig = binascii.unhexlify(sig) return base64.b64encode(sig) def ecdsa_verify(msg, sig, pub, usehex=True): - #See note to ecdsa_sign - hashed_msg = electrum_sig_hash(msg) + hashed_msg = message_sig_hash(msg) sig = base64.b64decode(sig) - #see comments to legacy* functions - sig = legacy_ecdsa_verify_convert(sig) if usehex: #arguments to raw_verify must be consistently hex or bin hashed_msg = binascii.hexlify(hashed_msg) sig = binascii.hexlify(sig) return ecdsa_raw_verify(hashed_msg, pub, sig, usehex, rawmsg=True) -#A sadly necessary hack until all joinmarket bots are running secp256k1 code. -#pybitcointools *message* signatures (not transaction signatures) used an old signature -#format, basically: [27+y%2] || 32 byte r || 32 byte s, -#instead of DER. These two functions translate the new version into the old so that -#counterparty bots can verify successfully. -def legacy_ecdsa_sign_convert(dersig): - #note there is no sanity checking of DER format (e.g. leading length byte) - dersig = dersig[2:] #e.g. 3045 - rlen = ord(dersig[1]) #ignore leading 02 - #length of r and s: ALWAYS <=33, USUALLY >=32 but can be shorter - if rlen > 33: - raise Exception("Incorrectly formatted DER sig:" + binascii.hexlify( - dersig)) - if dersig[2] == '\x00': - r = dersig[3:2 + rlen] - ssig = dersig[2 + rlen:] - else: - r = dersig[2:2 + rlen] - ssig = dersig[2 + rlen:] - - slen = ord(ssig[1]) #ignore leading 02 - if slen > 33: - raise Exception("Incorrectly formatted DER sig:" + binascii.hexlify( - dersig)) - if len(ssig) != 2 + slen: - raise Exception("Incorrectly formatted DER sig:" + binascii.hexlify( - dersig)) - if ssig[2] == '\x00': - s = ssig[3:2 + slen] - else: - s = ssig[2:2 + slen] - - #the legacy version requires padding of r and s to 32 bytes with leading zeros - r = '\x00' * (32 - len(r)) + r - s = '\x00' * (32 - len(s)) + s - - #note: in the original pybitcointools implementation, - #verification ignored the leading byte (it's only needed for pubkey recovery) - #so we just ignore parity here. - return chr(27) + r + s - -def legacy_ecdsa_verify_convert(sig): - sig = sig[1:] #ignore parity byte - r, s = sig[:32], sig[32:] - if not len(s) == 32: - #signature is invalid. - return False - #legacy code can produce high S. Need to reintroduce N ::cry:: - N = 115792089237316195423570985008687907852837564279074904382605163141518161494337 - s_int = decode(s, 256) - # note // is integer division operator in both 2.7 and 3 - s_int = N - s_int if s_int > N // 2 else s_int #enforce low S. - - #on re-encoding, don't use the minlen parameter, because - #DER does not used fixed (32 byte) length values, so we - #don't prepend zero bytes to shorter numbers. - s = encode(s_int, 256) - - #as above, remove any front zero padding from r. - r = encode(decode(r, 256), 256) - - #canonicalize r and s - r, s = ['\x00' + x if ord(x[0]) > 127 else x for x in [r, s]] - rlen = chr(len(r)) - slen = chr(len(s)) - total_len = 2 + len(r) + 2 + len(s) - return '\x30' + chr(total_len) + '\x02' + rlen + r + '\x02' + slen + s - #Use secp256k1 to handle all EC and ECDSA operations. #Data types: only hex and binary. #Compressed and uncompressed private and public keys. @@ -339,10 +299,14 @@ def ecdsa_raw_sign(msg, newpriv = secp256k1.PrivateKey(p, raw=True, ctx=ctx) else: newpriv = secp256k1.PrivateKey(priv, raw=False, ctx=ctx) - if usenonce and len(usenonce) != 32: - raise ValueError("Invalid nonce passed to ecdsa_sign: " + str(usenonce)) - - sig = newpriv.ecdsa_sign(msg, raw=rawmsg) + if usenonce: + if len(usenonce) != 32: + raise ValueError("Invalid nonce passed to ecdsa_sign: " + str( + usenonce)) + nf = ffi.addressof(_noncefunc.lib, "nonce_function_rand") + ndata = ffi.new("char [32]", usenonce) + usenonce = (nf, ndata) + sig = newpriv.ecdsa_sign(msg, raw=rawmsg, custom_nonce=usenonce) return newpriv.ecdsa_serialize(sig) @hexbin @@ -354,12 +318,19 @@ def ecdsa_raw_verify(msg, pub, sig, usehex, rawmsg=False): If rawmsg is False, the secp256k1 lib will hash the message as part of the ECDSA-SHA256 verification algo. Return value: True if the signature is valid for this pubkey, False - otherwise. ''' - if rawmsg and len(msg) != 32: - raise Exception("Invalid hash input to ECDSA raw sign.") - newpub = secp256k1.PublicKey(pubkey=pub, raw=True, ctx=ctx) - sigobj = newpub.ecdsa_deserialize(sig) - return newpub.ecdsa_verify(msg, sigobj, raw=rawmsg) + otherwise. + Since the arguments may come from external messages their content is + not guaranteed, so return False on any parsing exception. + ''' + try: + if rawmsg: + assert len(msg) == 32 + newpub = secp256k1.PublicKey(pubkey=pub, raw=True, ctx=ctx) + sigobj = newpub.ecdsa_deserialize(sig) + retval = newpub.ecdsa_verify(msg, sigobj, raw=rawmsg) + except: + return False + return retval def estimate_tx_size(ins, outs, txtype='p2pkh'): '''Estimate transaction size. diff --git a/bitcoin/transaction.py b/bitcoin/transaction.py deleted file mode 100644 index 4b9be322..00000000 --- a/bitcoin/transaction.py +++ /dev/null @@ -1,490 +0,0 @@ -#!/usr/bin/python -import binascii, re, json, copy, sys -from bitcoin.main import * -from _functools import reduce - -### Hex to bin converter and vice versa for objects - - -def json_is_base(obj, base): - if not is_python2 and isinstance(obj, bytes): - return False - - alpha = get_code_string(base) - if isinstance(obj, string_types): - for i in range(len(obj)): - if alpha.find(obj[i]) == -1: - return False - return True - elif isinstance(obj, int_types) or obj is None: - return True - elif isinstance(obj, list): - for i in range(len(obj)): - if not json_is_base(obj[i], base): - return False - return True - else: - for x in obj: - if not json_is_base(obj[x], base): - return False - return True - - -def json_changebase(obj, changer): - if isinstance(obj, string_or_bytes_types): - return changer(obj) - elif isinstance(obj, int_types) or obj is None: - return obj - elif isinstance(obj, list): - return [json_changebase(x, changer) for x in obj] - return dict((x, json_changebase(obj[x], changer)) for x in obj) - -# Transaction serialization and deserialization - - -def deserialize(tx): - if isinstance(tx, str) and re.match('^[0-9a-fA-F]*$', tx): - #tx = bytes(bytearray.fromhex(tx)) - return json_changebase( - deserialize(binascii.unhexlify(tx)), lambda x: safe_hexlify(x)) - # http://stackoverflow.com/questions/4851463/python-closure-write-to-variable-in-parent-scope - # Python's scoping rules are demented, requiring me to make pos an object - # so that it is call-by-reference - pos = [0] - - def read_as_int(bytez): - pos[0] += bytez - return decode(tx[pos[0] - bytez:pos[0]][::-1], 256) - - def read_var_int(): - pos[0] += 1 - - val = from_byte_to_int(tx[pos[0] - 1]) - if val < 253: - return val - return read_as_int(pow(2, val - 252)) - - def read_bytes(bytez): - pos[0] += bytez - return tx[pos[0] - bytez:pos[0]] - - def read_var_string(): - size = read_var_int() - return read_bytes(size) - - obj = {"ins": [], "outs": []} - obj["version"] = read_as_int(4) - ins = read_var_int() - for i in range(ins): - obj["ins"].append({ - "outpoint": { - "hash": read_bytes(32)[::-1], - "index": read_as_int(4) - }, - "script": read_var_string(), - "sequence": read_as_int(4) - }) - outs = read_var_int() - for i in range(outs): - obj["outs"].append({ - "value": read_as_int(8), - "script": read_var_string() - }) - obj["locktime"] = read_as_int(4) - return obj - - -def serialize(txobj): - #if isinstance(txobj, bytes): - # txobj = bytes_to_hex_string(txobj) - o = [] - if json_is_base(txobj, 16): - json_changedbase = json_changebase(txobj, - lambda x: binascii.unhexlify(x)) - hexlified = safe_hexlify(serialize(json_changedbase)) - return hexlified - o.append(encode(txobj["version"], 256, 4)[::-1]) - o.append(num_to_var_int(len(txobj["ins"]))) - for inp in txobj["ins"]: - o.append(inp["outpoint"]["hash"][::-1]) - o.append(encode(inp["outpoint"]["index"], 256, 4)[::-1]) - o.append(num_to_var_int(len(inp["script"])) + (inp["script"] if inp[ - "script"] or is_python2 else bytes())) - o.append(encode(inp["sequence"], 256, 4)[::-1]) - o.append(num_to_var_int(len(txobj["outs"]))) - for out in txobj["outs"]: - o.append(encode(out["value"], 256, 8)[::-1]) - o.append(num_to_var_int(len(out["script"])) + out["script"]) - o.append(encode(txobj["locktime"], 256, 4)[::-1]) - - return ''.join(o) if is_python2 else reduce(lambda x, y: x + y, o, bytes()) - -# Hashing transactions for signing - -SIGHASH_ALL = 1 -SIGHASH_NONE = 2 -SIGHASH_SINGLE = 3 -# this works like SIGHASH_ANYONECANPAY | SIGHASH_ALL, might as well make it explicit while -# we fix the constant -SIGHASH_ANYONECANPAY = 0x81 - - -def signature_form(tx, i, script, hashcode=SIGHASH_ALL): - i, hashcode = int(i), int(hashcode) - if isinstance(tx, string_or_bytes_types): - return serialize(signature_form(deserialize(tx), i, script, hashcode)) - newtx = copy.deepcopy(tx) - for inp in newtx["ins"]: - inp["script"] = "" - newtx["ins"][i]["script"] = script - if hashcode == SIGHASH_NONE: - newtx["outs"] = [] - elif hashcode == SIGHASH_SINGLE: - newtx["outs"] = newtx["outs"][:len(newtx["ins"])] - for out in range(len(newtx["ins"]) - 1): - out.value = 2**64 - 1 - out.script = "" - elif hashcode == SIGHASH_ANYONECANPAY: - newtx["ins"] = [newtx["ins"][i]] - else: - pass - return newtx - -# Making the actual signatures - - -def der_encode_sig(v, r, s): - """Takes (vbyte, r, s) as ints and returns hex der encode sig""" - #See https://github.com/vbuterin/pybitcointools/issues/89 - #See https://github.com/simcity4242/pybitcointools/ - s = N - s if s > N // 2 else s # BIP62 low s - b1, b2 = encode(r, 256), encode(s, 256) - if bytearray(b1)[ - 0] & 0x80: # add null bytes if leading byte interpreted as negative - b1 = b'\x00' + b1 - if bytearray(b2)[0] & 0x80: - b2 = b'\x00' + b2 - left = b'\x02' + encode(len(b1), 256, 1) + b1 - right = b'\x02' + encode(len(b2), 256, 1) + b2 - return safe_hexlify(b'\x30' + encode( - len(left + right), 256, 1) + left + right) - - -def der_decode_sig(sig): - leftlen = decode(sig[6:8], 16) * 2 - left = sig[8:8 + leftlen] - rightlen = decode(sig[10 + leftlen:12 + leftlen], 16) * 2 - right = sig[12 + leftlen:12 + leftlen + rightlen] - return (None, decode(left, 16), decode(right, 16)) - - -def txhash(tx, hashcode=None): - if isinstance(tx, str) and re.match('^[0-9a-fA-F]*$', tx): - tx = changebase(tx, 16, 256) - if hashcode: - return dbl_sha256(from_string_to_bytes(tx) + encode( - int(hashcode), 256, 4)[::-1]) - else: - return safe_hexlify(bin_dbl_sha256(tx)[::-1]) - - -def bin_txhash(tx, hashcode=None): - return binascii.unhexlify(txhash(tx, hashcode)) - - -def ecdsa_tx_sign(tx, priv, hashcode=SIGHASH_ALL): - rawsig = ecdsa_raw_sign(bin_txhash(tx, hashcode), priv) - return der_encode_sig(*rawsig) + encode(hashcode, 16, 2) - - -def ecdsa_tx_verify(tx, sig, pub, hashcode=SIGHASH_ALL): - return ecdsa_raw_verify(bin_txhash(tx, hashcode), der_decode_sig(sig), pub) - -# Scripts - -def mk_pubkey_script(addr): - # Keep the auxiliary functions around for altcoins' sake - return '76a914' + b58check_to_hex(addr) + '88ac' - - -def mk_scripthash_script(addr): - return 'a914' + b58check_to_hex(addr) + '87' - -# Address representation to output script - - -def address_to_script(addr): - if addr[0] == '3' or addr[0] == '2': - return mk_scripthash_script(addr) - else: - return mk_pubkey_script(addr) - -# Output script to address representation - - -def script_to_address(script, vbyte=0): - if re.match('^[0-9a-fA-F]*$', script): - script = binascii.unhexlify(script) - if script[:3] == b'\x76\xa9\x14' and script[-2:] == b'\x88\xac' and len( - script) == 25: - return bin_to_b58check(script[3:-2], vbyte) # pubkey hash addresses - else: - if vbyte in [111, 196]: - # Testnet - scripthash_byte = 196 - else: - scripthash_byte = 5 - # BIP0016 scripthash addresses - return bin_to_b58check(script[2:-1], scripthash_byte) - - -def p2sh_scriptaddr(script, magicbyte=5): - if re.match('^[0-9a-fA-F]*$', script): - script = binascii.unhexlify(script) - return hex_to_b58check(hash160(script), magicbyte) - - -scriptaddr = p2sh_scriptaddr - - -def deserialize_script(script): - if isinstance(script, str) and re.match('^[0-9a-fA-F]*$', script): - return json_changebase( - deserialize_script(binascii.unhexlify(script)), - lambda x: safe_hexlify(x)) - out, pos = [], 0 - while pos < len(script): - code = from_byte_to_int(script[pos]) - if code == 0: - out.append(None) - pos += 1 - elif code <= 75: - out.append(script[pos + 1:pos + 1 + code]) - pos += 1 + code - elif code <= 78: - szsz = pow(2, code - 76) - sz = decode(script[pos + szsz:pos:-1], 256) - out.append(script[pos + 1 + szsz:pos + 1 + szsz + sz]) - pos += 1 + szsz + sz - elif code <= 96: - out.append(code - 80) - pos += 1 - else: - out.append(code) - pos += 1 - return out - - -def serialize_script_unit(unit): - if isinstance(unit, int): - if unit < 16: - return from_int_to_byte(unit + 80) - else: - return bytes([unit]) - elif unit is None: - return b'\x00' - else: - if len(unit) <= 75: - return from_int_to_byte(len(unit)) + unit - elif len(unit) < 256: - return from_int_to_byte(76) + from_int_to_byte(len(unit)) + unit - elif len(unit) < 65536: - return from_int_to_byte(77) + encode(len(unit), 256, 2)[::-1] + unit - else: - return from_int_to_byte(78) + encode(len(unit), 256, 4)[::-1] + unit - - -if is_python2: - - def serialize_script(script): - if json_is_base(script, 16): - return binascii.hexlify(serialize_script(json_changebase( - script, lambda x: binascii.unhexlify(x)))) - return ''.join(map(serialize_script_unit, script)) -else: - - def serialize_script(script): - if json_is_base(script, 16): - return safe_hexlify(serialize_script(json_changebase( - script, lambda x: binascii.unhexlify(x)))) - - result = bytes() - for b in map(serialize_script_unit, script): - result += b if isinstance(b, bytes) else bytes(b, 'utf-8') - return result - - -def mk_multisig_script(*args): # [pubs],k or pub1,pub2...pub[n],k - if isinstance(args[0], list): - pubs, k = args[0], int(args[1]) - else: - pubs = list(filter(lambda x: len(str(x)) >= 32, args)) - k = int(args[len(pubs)]) - return serialize_script([k] + pubs + [len(pubs)]) + 'ae' - -# Signing and verifying - - -def verify_tx_input(tx, i, script, sig, pub): - if re.match('^[0-9a-fA-F]*$', tx): - tx = binascii.unhexlify(tx) - if re.match('^[0-9a-fA-F]*$', script): - script = binascii.unhexlify(script) - if not re.match('^[0-9a-fA-F]*$', sig): - sig = safe_hexlify(sig) - hashcode = decode(sig[-2:], 16) - modtx = signature_form(tx, int(i), script, hashcode) - return ecdsa_tx_verify(modtx, sig, pub, hashcode) - - -def sign(tx, i, priv, hashcode=SIGHASH_ALL): - i = int(i) - if (not is_python2 and isinstance(re, bytes)) or not re.match( - '^[0-9a-fA-F]*$', tx): - return binascii.unhexlify(sign(safe_hexlify(tx), i, priv)) - if len(priv) <= 33: - priv = safe_hexlify(priv) - pub = privkey_to_pubkey(priv) - address = pubkey_to_address(pub) - signing_tx = signature_form(tx, i, mk_pubkey_script(address), hashcode) - sig = ecdsa_tx_sign(signing_tx, priv, hashcode) - txobj = deserialize(tx) - txobj["ins"][i]["script"] = serialize_script([sig, pub]) - return serialize(txobj) - - -def signall(tx, priv): - # if priv is a dictionary, assume format is - # { 'txinhash:txinidx' : privkey } - if isinstance(priv, dict): - for e, i in enumerate(deserialize(tx)["ins"]): - k = priv["%s:%d" % (i["outpoint"]["hash"], i["outpoint"]["index"])] - tx = sign(tx, e, k) - else: - for i in range(len(deserialize(tx)["ins"])): - tx = sign(tx, i, priv) - return tx - - -def multisign(tx, i, script, pk, hashcode=SIGHASH_ALL): - if re.match('^[0-9a-fA-F]*$', tx): - tx = binascii.unhexlify(tx) - if re.match('^[0-9a-fA-F]*$', script): - script = binascii.unhexlify(script) - modtx = signature_form(tx, i, script, hashcode) - return ecdsa_tx_sign(modtx, pk, hashcode) - - -def apply_multisignatures(*args): - # tx,i,script,sigs OR tx,i,script,sig1,sig2...,sig[n] - tx, i, script = args[0], int(args[1]), args[2] - sigs = args[3] if isinstance(args[3], list) else list(args[3:]) - - if isinstance(script, str) and re.match('^[0-9a-fA-F]*$', script): - script = binascii.unhexlify(script) - sigs = [binascii.unhexlify(x) if x[:2] == '30' else x for x in sigs] - if isinstance(tx, str) and re.match('^[0-9a-fA-F]*$', tx): - return safe_hexlify(apply_multisignatures( - binascii.unhexlify(tx), i, script, sigs)) - - txobj = deserialize(tx) - txobj["ins"][i]["script"] = serialize_script([None] + sigs + [script]) - return serialize(txobj) - - -def is_inp(arg): - return len(arg) > 64 or "output" in arg or "outpoint" in arg - - -def mktx(*args): - # [in0, in1...],[out0, out1...] or in0, in1 ... out0 out1 ... - ins, outs = [], [] - for arg in args: - if isinstance(arg, list): - for a in arg: - (ins if is_inp(a) else outs).append(a) - else: - (ins if is_inp(arg) else outs).append(arg) - - txobj = {"locktime": 0, "version": 1, "ins": [], "outs": []} - for i in ins: - if isinstance(i, dict) and "outpoint" in i: - txobj["ins"].append(i) - else: - if isinstance(i, dict) and "output" in i: - i = i["output"] - txobj["ins"].append({ - "outpoint": {"hash": i[:64], - "index": int(i[65:])}, - "script": "", - "sequence": 4294967295 - }) - for o in outs: - if isinstance(o, string_or_bytes_types): - addr = o[:o.find(':')] - val = int(o[o.find(':') + 1:]) - o = {} - if re.match('^[0-9a-fA-F]*$', addr): - o["script"] = addr - else: - o["address"] = addr - o["value"] = val - - outobj = {} - if "address" in o: - outobj["script"] = address_to_script(o["address"]) - elif "script" in o: - outobj["script"] = o["script"] - else: - raise Exception("Could not find 'address' or 'script' in output.") - outobj["value"] = o["value"] - txobj["outs"].append(outobj) - - return serialize(txobj) - - -def select(unspent, value): - value = int(value) - high = [u for u in unspent if u["value"] >= value] - high.sort(key=lambda u: u["value"]) - low = [u for u in unspent if u["value"] < value] - low.sort(key=lambda u: -u["value"]) - if len(high): - return [high[0]] - i, tv = 0, 0 - while tv < value and i < len(low): - tv += low[i]["value"] - i += 1 - if tv < value: - raise Exception("Not enough funds") - return low[:i] - -# Only takes inputs of the form { "output": blah, "value": foo } - - -def mksend(*args): - argz, change, fee = args[:-2], args[-2], int(args[-1]) - ins, outs = [], [] - for arg in argz: - if isinstance(arg, list): - for a in arg: - (ins if is_inp(a) else outs).append(a) - else: - (ins if is_inp(arg) else outs).append(arg) - - isum = sum([i["value"] for i in ins]) - osum, outputs2 = 0, [] - for o in outs: - if isinstance(o, string_types): - o2 = {"address": o[:o.find(':')], "value": int(o[o.find(':') + 1:])} - else: - o2 = o - outputs2.append(o2) - osum += o2["value"] - - if isum < osum + fee: - raise Exception("Not enough money") - elif isum > osum + fee + 5430: - outputs2 += [{"address": change, "value": isum - osum - fee}] - - return mktx(ins, outputs2) diff --git a/broadcast-tx.py b/broadcast-tx.py index 83f1df85..59183f85 100644 --- a/broadcast-tx.py +++ b/broadcast-tx.py @@ -7,9 +7,9 @@ import time from joinmarket import OrderbookWatch, load_program_config, IRCMessageChannel -from joinmarket import jm_single +from joinmarket import jm_single, MessageChannelCollection from joinmarket import random_nick -from joinmarket import get_log, debug_dump_object +from joinmarket import get_log, debug_dump_object, get_irc_mchannels log = get_log() @@ -71,11 +71,12 @@ def main(): load_program_config() jm_single().nickname = random_nick() log.debug('starting broadcast-tx') - irc = IRCMessageChannel(jm_single().nickname) - taker = Broadcaster(irc, options.waittime, txhex) + mcs = [IRCMessageChannel(c, jm_single().nickname) for c in get_irc_mchannels()] + mcc = MessageChannelCollection(mcs) + taker = Broadcaster(mcc, options.waittime, txhex) try: - log.debug('starting irc') - irc.run() + log.debug('starting message channels') + mcc.run() except: log.debug('CRASHING, DUMPING EVERYTHING') debug_dump_object(taker) diff --git a/cmttools/add-utxo.py b/cmttools/add-utxo.py new file mode 100644 index 00000000..108bc664 --- /dev/null +++ b/cmttools/add-utxo.py @@ -0,0 +1,229 @@ +#! /usr/bin/env python +from __future__ import absolute_import +"""A very simple command line tool to import utxos to be used +as commitments into joinmarket's commitments.json file, allowing +users to retry transactions more often without getting banned by +the anti-snooping feature employed by makers. +""" + +import binascii +import sys +import os +import json +from pprint import pformat + +#needed until Jmkt is a package +script_dir = os.path.dirname(os.path.realpath(__file__)) +sys.path.insert(0, os.path.dirname(script_dir)) + +from optparse import OptionParser +import bitcoin as btc +from joinmarket import load_program_config, jm_single, get_p2pk_vbyte +from joinmarket import Wallet +from commitment_utils import get_utxo_info, validate_utxo_data, quit + +def add_external_commitments(utxo_datas): + """Persist the PoDLE commitments for this utxo + to the commitments.json file. The number of separate + entries is dependent on the taker_utxo_retries entry, by + default 3. + """ + def generate_single_podle_sig(u, priv, i): + """Make a podle entry for key priv at index i, using a dummy utxo value. + This calls the underlying 'raw' code based on the class PoDLE, not the + library 'generate_podle' which intelligently searches and updates commitments. + """ + #Convert priv to hex + hexpriv = btc.from_wif_privkey(priv, vbyte=get_p2pk_vbyte()) + podle = btc.PoDLE(u, hexpriv) + r = podle.generate_podle(i) + return (r['P'], r['P2'], r['sig'], + r['e'], r['commit']) + ecs = {} + for u, priv in utxo_datas: + ecs[u] = {} + ecs[u]['reveal']={} + for j in range(jm_single().config.getint("POLICY", "taker_utxo_retries")): + P, P2, s, e, commit = generate_single_podle_sig(u, priv, j) + if 'P' not in ecs[u]: + ecs[u]['P']=P + ecs[u]['reveal'][j] = {'P2':P2, 's':s, 'e':e} + btc.add_external_commitments(ecs) + +def main(): + parser = OptionParser( + usage= + 'usage: %prog [options] [txid:n]', + description="Adds one or more utxos to the list that can be used to make " + "commitments for anti-snooping. Note that this utxo, and its " + "PUBkey, will be revealed to makers, so consider the privacy " + "implication. " + + "It may be useful to those who are having trouble making " + "coinjoins due to several unsuccessful attempts (especially " + "if your joinmarket wallet is new). " + + "'Utxo' means unspent transaction output, it must not " + "already be spent. " + "The options -w, -r and -R offer ways to load these utxos " + "from a file or wallet. " + "If you enter a single utxo without these options, you will be " + "prompted to enter the private key here - it must be in " + "WIF compressed format. " + + "BE CAREFUL about handling private keys! " + "Don't do this in insecure environments. " + + "Also note this ONLY works for standard (p2pkh) utxos." + ) + parser.add_option( + '-r', + '--read-from-file', + action='store', + type='str', + dest='in_file', + help='name of plain text csv file containing utxos, one per line, format: ' + 'txid:N, WIF-compressed-privkey' + ) + parser.add_option( + '-R', + '--read-from-json', + action='store', + type='str', + dest='in_json', + help='name of json formatted file containing utxos with private keys, as ' + 'output from "python wallet-tool.py -u -p walletname showutxos"' + ) + parser.add_option( + '-w', + '--load-wallet', + action='store', + type='str', + dest='loadwallet', + help='name of wallet from which to load utxos and use as commitments.' + ) + parser.add_option( + '-g', + '--gap-limit', + action='store', + type='int', + dest='gaplimit', + default = 6, + help='Only to be used with -w; gap limit for Joinmarket wallet, default 6.' + ) + parser.add_option( + '-M', + '--max-mixdepth', + action='store', + type='int', + dest='maxmixdepth', + default=5, + help='Only to be used with -w; number of mixdepths for wallet, default 5.' + ) + parser.add_option( + '-d', + '--delete-external', + action='store_true', + dest='delete_ext', + help='deletes the current list of external commitment utxos', + default=False + ) + parser.add_option( + '-v', + '--validate-utxos', + action='store_true', + dest='validate', + help='validate the utxos and pubkeys provided against the blockchain', + default=False + ) + parser.add_option( + '-o', + '--validate-only', + action='store_true', + dest='vonly', + help='only validate the provided utxos (file or command line), not add', + default=False + ) + (options, args) = parser.parse_args() + load_program_config() + utxo_data = [] + if options.delete_ext: + other = options.in_file or options.in_json or options.loadwallet + if len(args) > 0 or other: + if raw_input("You have chosen to delete commitments, other arguments " + "will be ignored; continue? (y/n)") != 'y': + print "Quitting" + sys.exit(0) + c, e = btc.get_podle_commitments() + print pformat(e) + if raw_input( + "You will remove the above commitments; are you sure? (y/n): ") != 'y': + print "Quitting" + sys.exit(0) + btc.update_commitments(external_to_remove=e) + print "Commitments deleted." + sys.exit(0) + + #Three options (-w, -r, -R) for loading utxo and privkey pairs from a wallet, + #csv file or json file. + if options.loadwallet: + os.chdir('..') #yuck (see earlier comment about package) + wallet = Wallet(options.loadwallet, + options.maxmixdepth, + options.gaplimit) + os.chdir(os.path.join(os.getcwd(), 'cmttools')) + jm_single().bc_interface.sync_wallet(wallet) + unsp = {} + for u, av in wallet.unspent.iteritems(): + addr = av['address'] + key = wallet.get_key_from_addr(addr) + wifkey = btc.wif_compressed_privkey(key, vbyte=get_p2pk_vbyte()) + unsp[u] = {'address': av['address'], + 'value': av['value'], 'privkey': wifkey} + for u, pva in unsp.iteritems(): + utxo_data.append((u, pva['privkey'])) + elif options.in_file: + with open(options.in_file, "rb") as f: + utxo_info = f.readlines() + for ul in utxo_info: + ul = ul.rstrip() + if ul: + u, priv = get_utxo_info(ul) + if not u: + quit(parser, "Failed to parse utxo info: " + str(ul)) + utxo_data.append((u, priv)) + elif options.in_json: + if not os.path.isfile(options.in_json): + print "File: " + options.in_json + " not found." + sys.exit(0) + with open(options.in_json, "rb") as f: + try: + utxo_json = json.loads(f.read()) + except: + print "Failed to read json from " + options.in_json + sys.exit(0) + for u, pva in utxo_json.iteritems(): + utxo_data.append((u, pva['privkey'])) + elif len(args) == 1: + u = args[0] + priv = raw_input( + 'input private key for ' + u + ', in WIF compressed format : ') + u, priv = get_utxo_info(','.join([u, priv])) + if not u: + quit(parser, "Failed to parse utxo info: " + u) + utxo_data.append((u, priv)) + else: + quit(parser, 'Invalid syntax') + if options.validate or options.vonly: + if not validate_utxo_data(utxo_data): + quit(parser, "Utxos did not validate, quitting") + if options.vonly: + sys.exit(0) + + #We are adding utxos to the external list + assert len(utxo_data) + add_external_commitments(utxo_data) + +if __name__ == "__main__": + main() + print('done') diff --git a/cmttools/commitment_utils.py b/cmttools/commitment_utils.py new file mode 100644 index 00000000..f0c08dbf --- /dev/null +++ b/cmttools/commitment_utils.py @@ -0,0 +1,63 @@ +import bitcoin as btc +import sys, os +#needed until Jmkt is a package +script_dir = os.path.dirname(os.path.realpath(__file__)) +sys.path.insert(0, os.path.dirname(script_dir)) + +from joinmarket import jm_single, get_p2pk_vbyte + +def quit(parser, errmsg): + parser.error(errmsg) + sys.exit(0) + +def get_utxo_info(upriv): + """Verify that the input string parses correctly as (utxo, priv) + and return that. + """ + try: + u, priv = upriv.split(',') + u = u.strip() + priv = priv.strip() + txid, n = u.split(':') + assert len(txid)==64 + assert len(n) in range(1, 4) + n = int(n) + assert n in range(256) + except: + #not sending data to stdout in case privkey info + print "Failed to parse utxo information for utxo" + try: + hexpriv = btc.from_wif_privkey(priv, vbyte=get_p2pk_vbyte()) + except: + print "failed to parse privkey, make sure it's WIF compressed format." + return u, priv + +def validate_utxo_data(utxo_datas, retrieve=False): + """For each txid: N, privkey, first + convert the privkey and convert to address, + then use the blockchain instance to look up + the utxo and check that its address field matches. + If retrieve is True, return the set of utxos and their values. + """ + results = [] + for u, priv in utxo_datas: + print 'validating this utxo: ' + str(u) + hexpriv = btc.from_wif_privkey(priv, vbyte=get_p2pk_vbyte()) + addr = btc.privkey_to_address(hexpriv, magicbyte=get_p2pk_vbyte()) + print 'claimed address: ' + addr + res = jm_single().bc_interface.query_utxo_set([u]) + print 'blockchain shows this data: ' + str(res) + if len(res) != 1: + print "utxo not found on blockchain: " + str(u) + return False + if res[0]['address'] != addr: + print "privkey corresponds to the wrong address for utxo: " + str(u) + print "blockchain returned address: " + res[0]['address'] + print "your privkey gave this address: " + addr + return False + if retrieve: + results.append((u, res[0]['value'])) + print 'all utxos validated OK' + if retrieve: + return results + return True \ No newline at end of file diff --git a/cmttools/sendtomany.py b/cmttools/sendtomany.py new file mode 100644 index 00000000..79accc5c --- /dev/null +++ b/cmttools/sendtomany.py @@ -0,0 +1,110 @@ +#! /usr/bin/env python +from __future__ import absolute_import +"""A simple command line tool to create a bunch +of utxos from one (thus giving more potential commitments +for a Joinmarket user, although of course it may be useful +for other reasons). +""" + +import binascii +import sys, os +#needed until Jmkt is a package +script_dir = os.path.dirname(os.path.realpath(__file__)) +sys.path.insert(0, os.path.dirname(script_dir)) +from optparse import OptionParser +from commitment_utils import get_utxo_info, validate_utxo_data, quit +import bitcoin as btc +from pprint import pformat +from joinmarket import load_program_config +from joinmarket.wallet import estimate_tx_fee +from joinmarket import jm_single, get_p2pk_vbyte, validate_address, get_log + +log = get_log() + +def sign(utxo, priv, destaddrs): + """Sign a tx sending the amount amt, from utxo utxo, + equally to each of addresses in list destaddrs, + after fees; the purpose is to create a large + number of utxos. + """ + results = validate_utxo_data([(utxo, priv)], retrieve=True) + if not results: + return False + assert results[0][0] == utxo + amt = results[0][1] + ins = [utxo] + estfee = estimate_tx_fee(1, len(destaddrs)) + outs = [] + share = int((amt - estfee) / len(destaddrs)) + fee = amt - share*len(destaddrs) + assert fee >= estfee + log.debug("Using fee: " + str(fee)) + for i, addr in enumerate(destaddrs): + outs.append({'address': addr, 'value': share}) + unsigned_tx = btc.mktx(ins, outs) + return btc.sign(unsigned_tx, 0, btc.from_wif_privkey( + priv, vbyte=get_p2pk_vbyte())) + +def main(): + parser = OptionParser( + usage= + 'usage: %prog [options] utxo destaddr1 destaddr2 ..', + description="For creating multiple utxos from one (for commitments in JM)." + "Provide a utxo in form txid:N that has some unspent coins;" + "Specify a list of destination addresses and the coins will" + "be split equally between them (after bitcoin fees)." + + "You'll be prompted to enter the private key for the utxo" + "during the run; it must be in WIF compressed format." + "After the transaction is completed, the utxo strings for" + + "the new outputs will be shown." + "Note that these utxos will not be ready for use as external" + + "commitments in Joinmarket until 5 confirmations have passed." + " BE CAREFUL about handling private keys!" + " Don't do this in insecure environments." + " Also note this ONLY works for standard (p2pkh) utxos." + ) + parser.add_option( + '-v', + '--validate-utxos', + action='store_true', + dest='validate', + help='validate the utxos and pubkeys provided against the blockchain', + default=False + ) + parser.add_option( + '-o', + '--validate-only', + action='store_true', + dest='vonly', + help='only validate the provided utxos (file or command line), not add', + default=False + ) + (options, args) = parser.parse_args() + load_program_config() + if len(args) < 2: + quit(parser, 'Invalid syntax') + u = args[0] + priv = raw_input( + 'input private key for ' + u + ', in WIF compressed format : ') + u, priv = get_utxo_info(','.join([u, priv])) + if not u: + quit(parser, "Failed to parse utxo info: " + u) + destaddrs = args[1:] + for d in destaddrs: + if not validate_address(d): + quit(parser, "Address was not valid; wrong network?: " + d) + txsigned = sign(u, priv, destaddrs) + log.debug("Got signed transaction:\n" + txsigned) + log.debug("Deserialized:") + log.debug(pformat(btc.deserialize(txsigned))) + if raw_input('Would you like to push to the network? (y/n):')[0] != 'y': + log.debug("You chose not to broadcast the transaction, quitting.") + return + jm_single().bc_interface.pushtx(txsigned) + +if __name__ == "__main__": + main() + print('done') diff --git a/create-unsigned-tx.py b/create-unsigned-tx.py index 4004e702..a8f4ea83 100644 --- a/create-unsigned-tx.py +++ b/create-unsigned-tx.py @@ -16,7 +16,8 @@ jm_single, get_p2pk_vbyte, random_nick from joinmarket import get_log, choose_sweep_orders, choose_orders, \ pick_order, cheapest_order_choose, weighted_order_choose -from joinmarket import AbstractWallet, IRCMessageChannel, debug_dump_object +from joinmarket import AbstractWallet, IRCMessageChannel, debug_dump_object, \ + MessageChannelCollection, get_irc_mchannels import bitcoin as btc import sendpayment @@ -78,24 +79,14 @@ def create_tx(self): change_addr = self.taker.changeaddr choose_orders_recover = self.sendpayment_choose_orders - auth_addr = self.taker.utxo_data[self.taker.auth_utxo]['address'] self.taker.start_cj(self.taker.wallet, cjamount, orders, utxos, self.taker.destaddr, change_addr, self.taker.options.txfee, self.finishcallback, - choose_orders_recover, auth_addr) + choose_orders_recover) def finishcallback(self, coinjointx): if coinjointx.all_responded: - # now sign it ourselves tx = btc.serialize(coinjointx.latest_tx) - for index, ins in enumerate(coinjointx.latest_tx['ins']): - utxo = ins['outpoint']['hash'] + ':' + str(ins['outpoint'][ - 'index']) - if utxo != self.taker.auth_utxo: - continue - addr = coinjointx.input_utxos[utxo]['address'] - tx = btc.sign(tx, index, - coinjointx.wallet.get_key_from_addr(addr)) print 'unsigned tx = \n\n' + tx + '\n' log.debug('created unsigned tx, ending') self.taker.msgchan.shutdown() @@ -145,11 +136,10 @@ def run(self): class CreateUnsignedTx(takermodule.Taker): - def __init__(self, msgchan, wallet, auth_utxo, cjamount, destaddr, + def __init__(self, msgchan, wallet, cjamount, destaddr, changeaddr, utxo_data, options, chooseOrdersFunc): takermodule.Taker.__init__(self, msgchan) self.wallet = wallet - self.auth_utxo = auth_utxo self.cjamount = cjamount self.destaddr = destaddr self.changeaddr = changeaddr @@ -164,8 +154,7 @@ def on_welcome(self): def main(): parser = OptionParser( - usage='usage: %prog [options] [auth utxo] [cjamount] [cjaddr] [' - 'changeaddr] [utxos..]', + usage='usage: %prog [options] [cjamount] [cjaddr] [changeaddr] [utxos..]', description=('Creates an unsigned coinjoin transaction. Outputs ' 'a partially signed transaction hex string. The user ' 'must sign their inputs independently and broadcast ' @@ -224,14 +213,14 @@ def main(): # + ' command line in the format txid:output/value-in-satoshi') (options, args) = parser.parse_args() - if len(args) < 3: - parser.error('Needs a wallet, amount and destination address') + if len(args) < 4: + parser.error( + 'Needs an amount, destination address, change address and utxos ') sys.exit(0) - auth_utxo = args[0] - cjamount = int(args[1]) - destaddr = args[2] - changeaddr = args[3] - cold_utxos = args[4:] + cjamount = int(args[0]) + destaddr = args[1] + changeaddr = args[2] + cold_utxos = args[3:] load_program_config() addr_valid1, errormsg1 = validate_address(destaddr) @@ -248,21 +237,13 @@ def main(): print 'ERROR: Address invalid. ' + errormsg2 return - all_utxos = [auth_utxo] + cold_utxos - query_result = jm_single().bc_interface.query_utxo_set(all_utxos) + query_result = jm_single().bc_interface.query_utxo_set(cold_utxos) if None in query_result: print query_result utxo_data = {} - for utxo, data in zip(all_utxos, query_result): + for utxo, data in zip(cold_utxos, query_result): utxo_data[utxo] = {'address': data['address'], 'value': data['value']} - auth_privkey = raw_input('input private key for ' + utxo_data[auth_utxo][ - 'address'] + ' :') - if utxo_data[auth_utxo]['address'] != btc.privtoaddr( - auth_privkey, - magicbyte=get_p2pk_vbyte()): - print 'ERROR: privkey does not match auth utxo' - return - + print("Got this utxo data: " + str(utxo_data)) if options.pickorders and cjamount != 0: # cant use for sweeping chooseOrdersFunc = pick_order elif options.choosecheapest: @@ -270,24 +251,17 @@ def main(): else: # choose randomly (weighted) chooseOrdersFunc = weighted_order_choose - jm_single().nickname = random_nick() log.debug('starting sendpayment') - class UnsignedTXWallet(AbstractWallet): - - def get_key_from_addr(self, addr): - log.debug('getting privkey of ' + addr) - if btc.privtoaddr(auth_privkey, magicbyte=get_p2pk_vbyte()) != addr: - raise RuntimeError('privkey doesnt match given address') - return auth_privkey - - wallet = UnsignedTXWallet() - irc = IRCMessageChannel(jm_single().nickname) - taker = CreateUnsignedTx(irc, wallet, auth_utxo, cjamount, destaddr, + wallet = AbstractWallet() + wallet.unspent = None + mcs = [IRCMessageChannel(c, jm_single().nickname) for c in get_irc_mchannels()] + mcc = MessageChannelCollection(mcs) + taker = CreateUnsignedTx(mcc, wallet, cjamount, destaddr, changeaddr, utxo_data, options, chooseOrdersFunc) try: - log.debug('starting irc') - irc.run() + log.debug('starting message channels') + mcc.run() except: log.debug('CRASHING, DUMPING EVERYTHING') debug_dump_object(wallet, ['addr_cache', 'keys', 'wallet_name', 'seed']) diff --git a/joinmarket/__init__.py b/joinmarket/__init__.py index c5c04f12..c8232e21 100644 --- a/joinmarket/__init__.py +++ b/joinmarket/__init__.py @@ -11,15 +11,17 @@ from .irc import IRCMessageChannel, random_nick, B_PER_SEC from .jsonrpc import JsonRpcError, JsonRpcConnectionError, JsonRpc from .maker import Maker -from .message_channel import MessageChannel +from .message_channel import MessageChannel, MessageChannelCollection from .old_mnemonic import mn_decode, mn_encode from .slowaes import decryptData, encryptData from .taker import Taker, OrderbookWatch, CoinJoinTX from .wallet import AbstractWallet, BitcoinCoreInterface, Wallet, \ BitcoinCoreWallet from .configure import load_program_config, jm_single, get_p2pk_vbyte, \ - get_network, jm_single, get_network, validate_address + get_network, jm_single, get_network, validate_address, get_irc_mchannels, \ + check_utxo_blacklist from .blockchaininterface import BlockrInterface, BlockchainInterface +from .yieldgenerator import YieldGenerator, ygmain # Set default logging handler to avoid "No handler found" warnings. try: diff --git a/joinmarket/blockchaininterface.py b/joinmarket/blockchaininterface.py index b8b79842..08df7ace 100644 --- a/joinmarket/blockchaininterface.py +++ b/joinmarket/blockchaininterface.py @@ -89,8 +89,12 @@ def sync_unspent(self, wallet): sets wallet.unspent """ @abc.abstractmethod - def add_tx_notify(self, txd, unconfirmfun, confirmfun, - notifyaddr, timeoutfun=None): + def add_tx_notify(self, + txd, + unconfirmfun, + confirmfun, + notifyaddr, + timeoutfun=None): """ Invokes unconfirmfun and confirmfun when tx is seen on the network If timeoutfun not None, called with boolean argument that tells @@ -105,14 +109,15 @@ def pushtx(self, txhex): pass @abc.abstractmethod - def query_utxo_set(self, txouts): + def query_utxo_set(self, txouts, includeconf=False): """ takes a utxo or a list of utxos returns None if they are spend or unconfirmed otherwise returns value in satoshis, address and output script + optionally return the coin age in number of blocks """ # address and output script contain the same information btw - + @abc.abstractmethod def estimate_fee_per_kb(self, N): '''Use the blockchain interface to @@ -140,8 +145,8 @@ def sync_addresses(self, wallet): unused_addr_count = 0 last_used_addr = '' while (unused_addr_count < wallet.gaplimit or - not is_index_ahead_of_cache( - wallet, mix_depth, forchange)): + not is_index_ahead_of_cache(wallet, mix_depth, + forchange)): addrs = [wallet.get_new_addr(mix_depth, forchange) for _ in range(self.BLOCKR_MAX_ADDR_REQ_COUNT)] @@ -161,9 +166,9 @@ def sync_addresses(self, wallet): if last_used_addr == '': wallet.index[mix_depth][forchange] = 0 else: - next_avail_idx = max([wallet.addr_cache[last_used_addr][2]+1, - wallet.index_cache[mix_depth][forchange]]) - wallet.index[mix_depth][forchange] = next_avail_idx + next_avail_idx = max([wallet.addr_cache[last_used_addr][ + 2] + 1, wallet.index_cache[mix_depth][forchange]]) + wallet.index[mix_depth][forchange] = next_avail_idx def sync_unspent(self, wallet): # finds utxos in the wallet @@ -172,8 +177,8 @@ def sync_unspent(self, wallet): rate_limit_time = 10 * 60 if st - self.last_sync_unspent < rate_limit_time: log.debug( - 'blockr sync_unspent() happened too recently (%dsec), skipping' - % (st - self.last_sync_unspent)) + 'blockr sync_unspent() happened too recently (%dsec), skipping' + % (st - self.last_sync_unspent)) return wallet.unspent = {} @@ -201,25 +206,32 @@ def sync_unspent(self, wallet): for u in dat['unspent']: wallet.unspent[u['tx'] + ':' + str(u['n'])] = { 'address': dat['address'], - 'value': int(u['amount'].replace('.', ''))} + 'value': int(u['amount'].replace('.', '')) + } for u in wallet.spent_utxos: wallet.unspent.pop(u, None) self.last_sync_unspent = time.time() - log.debug('blockr sync_unspent took ' + str((self.last_sync_unspent - - st)) + 'sec') - - def add_tx_notify(self, txd, unconfirmfun, confirmfun, notifyaddr, timeoutfun=None): + log.debug('blockr sync_unspent took ' + str((self.last_sync_unspent - st + )) + 'sec') + + def add_tx_notify(self, + txd, + unconfirmfun, + confirmfun, + notifyaddr, + timeoutfun=None): unconfirm_timeout = jm_single().config.getint('TIMEOUT', - 'unconfirm_timeout_sec') + 'unconfirm_timeout_sec') unconfirm_poll_period = 5 - confirm_timeout = jm_single().config.getfloat('TIMEOUT', - 'confirm_timeout_hours')*60*60 + confirm_timeout = jm_single().config.getfloat( + 'TIMEOUT', 'confirm_timeout_hours') * 60 * 60 confirm_poll_period = 5 * 60 class NotifyThread(threading.Thread): - def __init__(self, blockr_domain, txd, unconfirmfun, confirmfun, timeoutfun): + def __init__(self, blockr_domain, txd, unconfirmfun, confirmfun, + timeoutfun): threading.Thread.__init__(self, name='BlockrNotifyThread') self.daemon = True self.blockr_domain = blockr_domain @@ -230,7 +242,8 @@ def __init__(self, blockr_domain, txd, unconfirmfun, confirmfun, timeoutfun): for sv in txd['outs']]) self.output_addresses = [ btc.script_to_address(scrval[0], get_p2pk_vbyte()) - for scrval in self.tx_output_set] + for scrval in self.tx_output_set + ] log.debug('txoutset=' + pprint.pformat(self.tx_output_set)) log.debug('outaddrs=' + ','.join(self.output_addresses)) @@ -248,11 +261,9 @@ def run(self): blockr_url = 'https://' + self.blockr_domain blockr_url += '.blockr.io/api/v1/address/unspent/' random.shuffle(self.output_addresses - ) # seriously weird bug with blockr.io - data = json.loads( - btc.make_request(blockr_url + ','.join( - self.output_addresses - ) + '?unconfirmed=1'))['data'] + ) # seriously weird bug with blockr.io + data = json.loads(btc.make_request(blockr_url + ','.join( + self.output_addresses) + '?unconfirmed=1'))['data'] shared_txid = None for unspent_list in data: @@ -266,12 +277,12 @@ def run(self): if len(shared_txid) == 0: continue time.sleep( - 2 + 2 ) # here for some race condition bullshit with blockr.io blockr_url = 'https://' + self.blockr_domain blockr_url += '.blockr.io/api/v1/tx/raw/' data = json.loads(btc.make_request(blockr_url + ','.join( - shared_txid)))['data'] + shared_txid)))['data'] if not isinstance(data, list): data = [data] for txinfo in data: @@ -285,7 +296,7 @@ def run(self): break self.unconfirmfun( - btc.deserialize(unconfirmed_txhex), unconfirmed_txid) + btc.deserialize(unconfirmed_txhex), unconfirmed_txid) st = int(time.time()) confirmed_txid = None @@ -300,7 +311,7 @@ def run(self): blockr_url = 'https://' + self.blockr_domain blockr_url += '.blockr.io/api/v1/address/txs/' data = json.loads(btc.make_request(blockr_url + ','.join( - self.output_addresses)))['data'] + self.output_addresses)))['data'] shared_txid = None for addrtxs in data: txs = set(str(txdata['tx']) @@ -314,9 +325,8 @@ def run(self): continue blockr_url = 'https://' + self.blockr_domain blockr_url += '.blockr.io/api/v1/tx/raw/' - data = json.loads( - btc.make_request( - blockr_url + ','.join(shared_txid)))['data'] + data = json.loads(btc.make_request(blockr_url + ','.join( + shared_txid)))['data'] if not isinstance(data, list): data = [data] for txinfo in data: @@ -329,9 +339,10 @@ def run(self): confirmed_txhex = str(txinfo['tx']['hex']) break self.confirmfun( - btc.deserialize(confirmed_txhex), confirmed_txid, 1) + btc.deserialize(confirmed_txhex), confirmed_txid, 1) - NotifyThread(self.blockr_domain, txd, unconfirmfun, confirmfun, timeoutfun).start() + NotifyThread(self.blockr_domain, txd, unconfirmfun, confirmfun, + timeoutfun).start() def pushtx(self, txhex): try: @@ -346,7 +357,7 @@ def pushtx(self, txhex): return False return True - def query_utxo_set(self, txout): + def query_utxo_set(self, txout, includeconf=False): if not isinstance(txout, list): txout = [txout] txids = [h[:64] for h in txout] @@ -359,8 +370,8 @@ def query_utxo_set(self, txout): data = [] for ids in txids: blockr_url = 'https://' + self.blockr_domain + '.blockr.io/api/v1/tx/info/' - blockr_data = json.loads( - btc.make_request(blockr_url + ','.join(ids)))['data'] + blockr_data = json.loads(btc.make_request(blockr_url + ','.join( + ids)))['data'] if not isinstance(blockr_data, list): blockr_data = [blockr_data] data += blockr_data @@ -368,32 +379,37 @@ def query_utxo_set(self, txout): for txo in txout: txdata = [d for d in data if d['tx'] == txo[:64]][0] vout = [v for v in txdata['vouts'] if v['n'] == int(txo[65:])][0] - if vout['is_spent'] == 1: + if "is_spent" in vout and vout['is_spent'] == 1: result.append(None) else: - result.append({'value': int(Decimal(vout['amount']) * Decimal( - '1e8')), + result_dict = {'value': int(Decimal(vout['amount']) * Decimal( + '1e8')), 'address': vout['address'], - 'script': vout['extras']['script']}) + 'script': vout['extras']['script']} + if includeconf: + result_dict['confirms'] = int(txdata['confirmations']) + result.append(result_dict) return result def estimate_fee_per_kb(self, N): bcypher_fee_estimate_url = 'https://api.blockcypher.com/v1/btc/main' bcypher_data = json.loads(btc.make_request(bcypher_fee_estimate_url)) - log.debug("Got blockcypher result: "+pprint.pformat(bcypher_data)) - if N<=2: - fee_per_kb = bcypher_data["high_fee_per_kb"] - elif N <=4: - fee_per_kb = bcypher_data["medium_fee_per_kb"] - else: - fee_per_kb = bcypher_data["low_fee_per_kb"] - - return fee_per_kb + log.debug("Got blockcypher result: " + pprint.pformat(bcypher_data)) + if N <= 2: + fee_per_kb = bcypher_data["high_fee_per_kb"] + elif N <= 4: + fee_per_kb = bcypher_data["medium_fee_per_kb"] + else: + fee_per_kb = bcypher_data["low_fee_per_kb"] + + return fee_per_kb + def bitcoincore_timeout_callback(uc_called, txout_set, txnotify_fun_list, timeoutfun): - log.debug('bitcoin core timeout callback uc_called = %s' % ('true' if - uc_called else 'false')) + log.debug('bitcoin core timeout callback uc_called = %s' % ('true' + if uc_called + else 'false')) txnotify_tuple = None for tnf in txnotify_fun_list: if tnf[0] == txout_set and uc_called == tnf[-1]: @@ -406,12 +422,14 @@ def bitcoincore_timeout_callback(uc_called, txout_set, txnotify_fun_list, log.debug('timeoutfun txout_set=\n' + pprint.pformat(txout_set)) timeoutfun(uc_called) + class NotifyRequestHeader(BaseHTTPServer.BaseHTTPRequestHandler): + def __init__(self, request, client_address, base_server): self.btcinterface = base_server.btcinterface self.base_server = base_server BaseHTTPServer.BaseHTTPRequestHandler.__init__( - self, request, client_address, base_server) + self, request, client_address, base_server) def do_HEAD(self): pages = ('/walletnotify?', '/alertnotify?') @@ -434,8 +452,8 @@ def do_HEAD(self): 'outs']]) txnotify_tuple = None - unconfirmfun, confirmfun, timeoutfun, uc_called = (None, None, - None, None) + unconfirmfun, confirmfun, timeoutfun, uc_called = (None, None, None, + None) for tnf in self.btcinterface.txnotify_fun: tx_out = tnf[0] if tx_out == tx_output_set: @@ -460,34 +478,37 @@ def do_HEAD(self): # amount = # bitcoin-cli move wallet_name "" amount self.btcinterface.txnotify_fun.remove(txnotify_tuple) - self.btcinterface.txnotify_fun.append(txnotify_tuple[:-1] - + (True,)) + self.btcinterface.txnotify_fun.append(txnotify_tuple[:-1] + + (True,)) log.debug('ran unconfirmfun') if timeoutfun: - threading.Timer(jm_single().config.getfloat('TIMEOUT', - 'confirm_timeout_hours')*60*60, - bitcoincore_timeout_callback, - args=(True, tx_output_set, - self.btcinterface.txnotify_fun, - timeoutfun)).start() + threading.Timer(jm_single().config.getfloat( + 'TIMEOUT', 'confirm_timeout_hours') * 60 * 60, + bitcoincore_timeout_callback, + args=(True, tx_output_set, + self.btcinterface.txnotify_fun, + timeoutfun)).start() else: if not uc_called: unconfirmfun(txd, txid) log.debug('saw confirmed tx before unconfirmed, ' + - 'running unconfirmfun first') + 'running unconfirmfun first') confirmfun(txd, txid, txdata['confirmations']) self.btcinterface.txnotify_fun.remove(txnotify_tuple) log.debug('ran confirmfun') elif self.path.startswith('/alertnotify?'): - jm_single().core_alert[0] = urllib.unquote(self.path[len(pages[1]):]) + jm_single().core_alert[0] = urllib.unquote(self.path[len(pages[ + 1]):]) log.debug('Got an alert!\nMessage=' + jm_single().core_alert[0]) else: - log.debug('ERROR: This is not a handled URL path. You may want to check your notify URL for typos.') + log.debug( + 'ERROR: This is not a handled URL path. You may want to check your notify URL for typos.') - request = urllib2.Request('http://localhost:' + str(self.base_server.server_address[1] + 1) + self.path) - request.get_method = lambda : 'HEAD' + request = urllib2.Request('http://localhost:' + str( + self.base_server.server_address[1] + 1) + self.path) + request.get_method = lambda: 'HEAD' try: urllib2.urlopen(request) except urllib2.URLError: @@ -498,6 +519,7 @@ def do_HEAD(self): class BitcoinCoreNotifyThread(threading.Thread): + def __init__(self, btcinterface): threading.Thread.__init__(self, name='CoreNotifyThread') self.daemon = True @@ -523,12 +545,13 @@ def run(self): httpd.serve_forever() log.debug('failed to bind for bitcoin core notify listening') - # must run bitcoind with -server # -walletnotify="curl -sI --connect-timeout 1 http://localhost:62602/walletnotify?%s" # and make sure curl is installed (git uses it, odds are you've already got it) + class BitcoinCoreInterface(BlockchainInterface): + def __init__(self, jsonRpc, network): super(BitcoinCoreInterface, self).__init__() self.jsonRpc = jsonRpc @@ -561,8 +584,8 @@ def add_watchonly_addresses(self, addr_list, wallet_name): ' addresses into account ' + wallet_name) for addr in addr_list: self.rpc('importaddress', [addr, wallet_name, False]) - if jm_single().config.get( - "BLOCKCHAIN", "blockchain_source") != 'regtest': + if jm_single().config.get("BLOCKCHAIN", + "blockchain_source") != 'regtest': print('restart Bitcoin Core with -rescan if you\'re ' 'recovering an existing wallet from backup seed') print(' otherwise just restart this joinmarket script') @@ -575,43 +598,44 @@ def sync_addresses(self, wallet): return log.debug('requesting wallet history') wallet_name = self.get_wallet_name(wallet) - #TODO It is worth considering making this user configurable: + #TODO It is worth considering making this user configurable: addr_req_count = 20 wallet_addr_list = [] for mix_depth in range(wallet.max_mix_depth): for forchange in [0, 1]: - #If we have an index-cache available, we can use it - #to decide how much to import (note that this list - #*always* starts from index 0 on each branch). - #In cases where the Bitcoin Core instance is fresh, - #this will allow the entire import+rescan to occur - #in 2 steps only. - if wallet.index_cache != [[0,0]]*wallet.max_mix_depth: - #Need to request N*addr_req_count where N is least s.t. - #N*addr_req_count > index_cache val. This is so that the batching - #process in the main loop *always* has already imported enough - #addresses to complete. - req_count = int(wallet.index_cache[mix_depth] - [forchange]/addr_req_count) + 1 - req_count *= addr_req_count - else: - #If we have *nothing* - no index_cache, and no info - #in Core wallet (imports), we revert to a batching mode - #with a default size. - #In this scenario it could require several restarts *and* - #rescans; perhaps user should set addr_req_count high - #(see above TODO) - req_count = addr_req_count + #If we have an index-cache available, we can use it + #to decide how much to import (note that this list + #*always* starts from index 0 on each branch). + #In cases where the Bitcoin Core instance is fresh, + #this will allow the entire import+rescan to occur + #in 2 steps only. + if wallet.index_cache != [[0, 0]] * wallet.max_mix_depth: + #Need to request N*addr_req_count where N is least s.t. + #N*addr_req_count > index_cache val. This is so that the batching + #process in the main loop *always* has already imported enough + #addresses to complete. + req_count = int(wallet.index_cache[mix_depth][forchange] / + addr_req_count) + 1 + req_count *= addr_req_count + else: + #If we have *nothing* - no index_cache, and no info + #in Core wallet (imports), we revert to a batching mode + #with a default size. + #In this scenario it could require several restarts *and* + #rescans; perhaps user should set addr_req_count high + #(see above TODO) + req_count = addr_req_count wallet_addr_list += [wallet.get_new_addr(mix_depth, forchange) for _ in range(req_count)] - #Indices are reset here so that the next algorithm step starts - #from the beginning of each branch + #Indices are reset here so that the next algorithm step starts + #from the beginning of each branch wallet.index[mix_depth][forchange] = 0 # makes more sense to add these in an account called "joinmarket-imported" but its much # simpler to add to the same account here for privkey_list in wallet.imported_privkeys.values(): for privkey in privkey_list: - imported_addr = btc.privtoaddr(privkey, magicbyte=get_p2pk_vbyte()) + imported_addr = btc.privtoaddr(privkey, + magicbyte=get_p2pk_vbyte()) wallet_addr_list.append(imported_addr) imported_addr_list = self.rpc('getaddressesbyaccount', [wallet_name]) if not set(wallet_addr_list).issubset(set(imported_addr_list)): @@ -646,11 +670,12 @@ def sync_addresses(self, wallet): break mix_change_addrs = [ wallet.get_new_addr(mix_depth, forchange) - for _ in range(addr_req_count)] + for _ in range(addr_req_count) + ] for mc_addr in mix_change_addrs: if mc_addr not in imported_addr_list: - too_few_addr_mix_change.append( - (mix_depth, forchange)) + too_few_addr_mix_change.append((mix_depth, forchange + )) breakloop = True break if mc_addr in used_addr_list: @@ -658,37 +683,38 @@ def sync_addresses(self, wallet): unused_addr_count = 0 else: unused_addr_count += 1 - #index setting here depends on whether we broke out of the loop - #early; if we did, it means we need to prepare the index - #at the level of the last used address or zero so as to not - #miss any imports in add_watchonly_addresses. - #If we didn't, we need to respect the index_cache to avoid - #potential address reuse. - if breakloop: - if last_used_addr == '': - wallet.index[mix_depth][forchange] = 0 - else: - wallet.index[mix_depth][forchange] = \ - wallet.addr_cache[last_used_addr][2] + 1 - else: - if last_used_addr == '': - next_avail_idx = max([wallet.index_cache[mix_depth] - [forchange], 0]) - else: - next_avail_idx = max([wallet.addr_cache[last_used_addr] - [2]+1, wallet.index_cache[mix_depth] - [forchange]]) - wallet.index[mix_depth][forchange] = next_avail_idx +#index setting here depends on whether we broke out of the loop +#early; if we did, it means we need to prepare the index +#at the level of the last used address or zero so as to not +#miss any imports in add_watchonly_addresses. +#If we didn't, we need to respect the index_cache to avoid +#potential address reuse. + if breakloop: + if last_used_addr == '': + wallet.index[mix_depth][forchange] = 0 + else: + wallet.index[mix_depth][forchange] = \ + wallet.addr_cache[last_used_addr][2] + 1 + else: + if last_used_addr == '': + next_avail_idx = max([wallet.index_cache[mix_depth][ + forchange], 0]) + else: + next_avail_idx = max([wallet.addr_cache[last_used_addr][ + 2] + 1, wallet.index_cache[mix_depth][forchange]]) + wallet.index[mix_depth][forchange] = next_avail_idx wallet_addr_list = [] if len(too_few_addr_mix_change) > 0: - indices = [wallet.index[mc[0]][mc[1]] for mc in too_few_addr_mix_change] + indices = [wallet.index[mc[0]][mc[1]] + for mc in too_few_addr_mix_change] log.debug('too few addresses in ' + str(too_few_addr_mix_change) + ' at ' + str(indices)) for mix_depth, forchange in too_few_addr_mix_change: wallet_addr_list += [ wallet.get_new_addr(mix_depth, forchange) - for _ in range(addr_req_count * 3)] + for _ in range(addr_req_count * 3) + ] self.add_watchonly_addresses(wallet_addr_list, wallet_name) return @@ -706,8 +732,8 @@ def sync_unspent(self, wallet): listunspent_args = [] if 'listunspent_args' in jm_single().config.options('POLICY'): - listunspent_args = ast.literal_eval( - jm_single().config.get('POLICY', 'listunspent_args')) + listunspent_args = ast.literal_eval(jm_single().config.get( + 'POLICY', 'listunspent_args')) unspent_list = self.rpc('listunspent', listunspent_args) for u in unspent_list: @@ -719,12 +745,17 @@ def sync_unspent(self, wallet): continue wallet.unspent[u['txid'] + ':' + str(u['vout'])] = { 'address': u['address'], - 'value': int(Decimal(str(u['amount'])) * Decimal('1e8'))} + 'value': int(Decimal(str(u['amount'])) * Decimal('1e8')) + } et = time.time() log.debug('bitcoind sync_unspent took ' + str((et - st)) + 'sec') - def add_tx_notify(self, txd, unconfirmfun, confirmfun, notifyaddr, - timeoutfun=None): + def add_tx_notify(self, + txd, + unconfirmfun, + confirmfun, + notifyaddr, + timeoutfun=None): if not self.notifythread: self.notifythread = BitcoinCoreNotifyThread(self) self.notifythread.start() @@ -738,27 +769,28 @@ def add_tx_notify(self, txd, unconfirmfun, confirmfun, notifyaddr, self.rpc('importaddress', [notifyaddr, 'joinmarket-notify', False]) tx_output_set = set([(sv['script'], sv['value']) for sv in txd['outs']]) self.txnotify_fun.append((tx_output_set, unconfirmfun, confirmfun, - timeoutfun, False)) + timeoutfun, False)) #create unconfirm timeout here, create confirm timeout in the other thread if timeoutfun: threading.Timer(jm_single().config.getint('TIMEOUT', - 'unconfirm_timeout_sec'), bitcoincore_timeout_callback, - args=(False, tx_output_set, self.txnotify_fun, timeoutfun) - ).start() + 'unconfirm_timeout_sec'), + bitcoincore_timeout_callback, + args=(False, tx_output_set, self.txnotify_fun, + timeoutfun)).start() def pushtx(self, txhex): try: txid = self.rpc('sendrawtransaction', [txhex]) except JsonRpcConnectionError as e: log.debug('error pushing = ' + repr(e)) - return False + return False except JsonRpcError as e: log.debug('error pushing = ' + str(e.code) + " " + str(e.message)) - return False + return False return True - def query_utxo_set(self, txout): + def query_utxo_set(self, txout, includeconf=False): if not isinstance(txout, list): txout = [txout] result = [] @@ -767,48 +799,56 @@ def query_utxo_set(self, txout): if ret is None: result.append(None) else: - result.append({'value': int(Decimal(str(ret['value'])) * + result_dict = {'value': int(Decimal(str(ret['value'])) * Decimal('1e8')), 'address': ret['scriptPubKey']['addresses'][0], - 'script': ret['scriptPubKey']['hex']}) + 'script': ret['scriptPubKey']['hex']} + if includeconf: + result_dict['confirms'] = int(ret['confirmations']) + result.append(result_dict) return result def estimate_fee_per_kb(self, N): - estimate = Decimal(1e8)*Decimal(self.rpc('estimatefee', [N])) - if estimate < 0: - #This occurs when Core has insufficient data to estimate. - #TODO anything better than a hardcoded default? - return 30000 + estimate = Decimal(1e8) * Decimal(self.rpc('estimatefee', [N])) + if estimate < 0: + #This occurs when Core has insufficient data to estimate. + #TODO anything better than a hardcoded default? + return 30000 else: - return estimate + return estimate + # class for regtest chain access # running on local daemon. Only # to be instantiated after network is up # with > 100 blocks. class RegtestBitcoinCoreInterface(BitcoinCoreInterface): + def __init__(self, jsonRpc): super(RegtestBitcoinCoreInterface, self).__init__(jsonRpc, 'regtest') self.pushtx_failure_prob = 0 self.tick_forward_chain_interval = 2 - self.absurd_fees = False + self.absurd_fees = False def estimate_fee_per_kb(self, N): - if not self.absurd_fees: - return super(RegtestBitcoinCoreInterface, self).estimate_fee_per_kb(N) - else: - return jm_single().config.getint("POLICY", "absurd_fee_per_kb") + 100 + if not self.absurd_fees: + return super(RegtestBitcoinCoreInterface, + self).estimate_fee_per_kb(N) + else: + return jm_single().config.getint("POLICY", + "absurd_fee_per_kb") + 100 def pushtx(self, txhex): if self.pushtx_failure_prob != 0 and random.random() <\ self.pushtx_failure_prob: log.debug('randomly not broadcasting %0.1f%% of the time' % - (self.pushtx_failure_prob*100)) + (self.pushtx_failure_prob * 100)) return True ret = super(RegtestBitcoinCoreInterface, self).pushtx(txhex) class TickChainThread(threading.Thread): + def __init__(self, bcinterface): threading.Thread.__init__(self, name='TickChainThread') self.bcinterface = bcinterface @@ -828,15 +868,16 @@ def tick_forward_chain(self, n): Special method for regtest only; instruct to mine n blocks. """ - try: - self.rpc('generate', [n]) - except JsonRpcConnectionError: - #can happen if the blockchain is shut down - #automatically at the end of tests; this shouldn't - #trigger an error - log.debug("Failed to generate blocks, looks like the bitcoin daemon \ + try: + self.rpc('generate', [n]) + except JsonRpcConnectionError: + #can happen if the blockchain is shut down + #automatically at the end of tests; this shouldn't + #trigger an error + log.debug( + "Failed to generate blocks, looks like the bitcoin daemon \ has been shut down. Ignoring.") - pass + pass def grab_coins(self, receiving_addr, amt=50): """ @@ -872,8 +913,8 @@ def get_received_by_addr(self, addresses, query_params): for address in addresses: self.rpc('importaddress', [address, 'watchonly']) res.append({'address': address, - 'balance': int(Decimal(1e8) * Decimal( - self.rpc('getreceivedbyaddress', [address])))}) + 'balance': int(round(Decimal(1e8) * Decimal(self.rpc( + 'getreceivedbyaddress', [address]))))}) return {'data': res} # todo: won't run anyways diff --git a/joinmarket/configure.py b/joinmarket/configure.py index 0be85b51..3867e8ae 100644 --- a/joinmarket/configure.py +++ b/joinmarket/configure.py @@ -3,6 +3,9 @@ import io import logging import threading +import os +import binascii +import sys from ConfigParser import SafeConfigParser, NoOptionError @@ -10,9 +13,6 @@ from joinmarket.jsonrpc import JsonRpc from joinmarket.support import get_log, joinmarket_alert, core_alert, debug_silence -# config = SafeConfigParser() -# config_location = 'joinmarket.cfg' - log = get_log() @@ -37,55 +37,38 @@ def add_entries(self, **entries): def __setattr__(self, name, value): if name == 'nickname' and value: logFormatter = logging.Formatter( - ('%(asctime)s [%(threadName)-12.12s] ' - '[%(levelname)-5.5s] %(message)s')) - fileHandler = logging.FileHandler( - 'logs/{}.log'.format(value)) + ('%(asctime)s [%(threadName)-12.12s] ' + '[%(levelname)-5.5s] %(message)s')) + fileHandler = logging.FileHandler('logs/{}.log'.format(value)) fileHandler.setFormatter(logFormatter) log.addHandler(fileHandler) super(AttributeDict, self).__setattr__(name, value) - def __getitem__(self, key): """ Provides dict-style access to attributes """ return getattr(self, key) - -# global_singleton = AttributeDict( -# **{'log': log, -# 'JM_VERSION': 3, -# 'nickname': None, -# 'DUST_THRESHOLD': 2730, -# 'bc_interface': None, -# 'ordername_list': ["absorder", "relorder"], -# 'maker_timeout_sec': 30, -# 'debug_file_lock': threading.Lock(), -# 'debug_file_handle': None, -# 'core_alert': None, -# 'joinmarket_alert': None, -# 'debug_silence': False, -# 'config': SafeConfigParser(), -# 'config_location': 'joinmarket.cfg'}) - -# todo: same as above. decide!!! global_singleton = AttributeDict() -global_singleton.JM_VERSION = 4 +global_singleton.JM_VERSION = 5 global_singleton.nickname = None global_singleton.DUST_THRESHOLD = 2730 global_singleton.bc_interface = None -global_singleton.ordername_list = ['absorder', 'relorder'] +global_singleton.ordername_list = ['absoffer', 'reloffer'] +global_singleton.commitment_broadcast_list = ['hp2'] global_singleton.maker_timeout_sec = 60 global_singleton.debug_file_lock = threading.Lock() global_singleton.debug_file_handle = None +global_singleton.blacklist_file_lock = threading.Lock() global_singleton.core_alert = core_alert global_singleton.joinmarket_alert = joinmarket_alert global_singleton.debug_silence = debug_silence global_singleton.config = SafeConfigParser() global_singleton.config_location = 'joinmarket.cfg' - +global_singleton.commit_file_location = 'cmttools/commitments.json' +global_singleton.wait_for_commitments = 0 def jm_single(): return global_singleton @@ -93,7 +76,8 @@ def jm_single(): # FIXME: Add rpc_* options here in the future! required_options = {'BLOCKCHAIN': ['blockchain_source', 'network'], 'MESSAGING': ['host', 'channel', 'port'], - 'POLICY': ['absurd_fee_per_kb']} + 'POLICY': ['absurd_fee_per_kb', 'taker_utxo_retries', + 'taker_utxo_age', 'taker_utxo_amtpercent']} defaultconfig = \ """ @@ -163,17 +147,63 @@ def jm_single(): # spends from unconfirmed inputs, which may then get malleated or double-spent! # other counterparties are likely to reject unconfirmed inputs... don't do it. -tx_broadcast = self #options: self, random-peer, not-self, random-maker # self = broadcast transaction with your own ip # random-peer = everyone who took part in the coinjoin has a chance of broadcasting # not-self = never broadcast with your own ip # random-maker = every peer on joinmarket has a chance of broadcasting, including yourself +tx_broadcast = self + +#THE FOLLOWING SETTINGS ARE REQUIRED TO DEFEND AGAINST SNOOPERS. +#DON'T ALTER THEM UNLESS YOU UNDERSTAND THE IMPLICATIONS. + +# number of retries allowed for a specific utxo, to prevent DOS/snooping. +# Lower settings make snooping more expensive, but also prevent honest users +# from retrying if an error occurs. +taker_utxo_retries = 3 + +# number of confirmations required for the commitment utxo mentioned above. +# this effectively rate-limits a snooper. +taker_utxo_age = 5 + +# percentage of coinjoin amount that the commitment utxo must have +# as a minimum BTC amount. Thus 20 means a 1BTC coinjoin requires the +# utxo to be at least 0.2 btc. +taker_utxo_amtpercent = 20 + +#Set to 1 to accept broadcast PoDLE commitments from other bots, and +#add them to your blacklist (only relevant for Makers). +#There is no way to spoof these values, so the only "risk" is that +#someone fills your blacklist file with a lot of data. +accept_commitment_broadcasts = 1 + +#Location of your commitments.json file (stores commitments you've used +#and those you want to use in future), relative to root joinmarket directory. +commit_file_location = cmttools/commitments.json """ -def get_config_irc_channel(): - channel = '#' + global_singleton.config.get("MESSAGING", "channel") +def get_irc_mchannels(): + fields = [("host", str), ("port", int), ("channel", str), + ("usessl", str), ("socks5", str), ("socks5_host", str), + ("socks5_port", str)] + configdata = {} + for f, t in fields: + vals = jm_single().config.get("MESSAGING", f).split(",") + if t == str: + vals = [x.strip() for x in vals] + else: + vals = [t(x) for x in vals] + configdata[f] = vals + configs = [] + for i in range(len(configdata['host'])): + newconfig = dict([(x, configdata[x][i]) for x in configdata]) + configs.append(newconfig) + return configs + + +def get_config_irc_channel(channel_name): + channel = "#" + channel_name if get_network() == 'testnet': channel += '-test' return channel @@ -209,21 +239,66 @@ def validate_address(addr): return False, "Address has correct checksum but wrong length." return True, 'address validated' +def donation_address(reusable_donation_pubkey=None): + if not reusable_donation_pubkey: + reusable_donation_pubkey = ('02be838257fbfddabaea03afbb9f16e852' + '9dfe2de921260a5c46036d97b5eacf2a') + sign_k = binascii.hexlify(os.urandom(32)) + c = btc.sha256(btc.multiply(sign_k, + reusable_donation_pubkey, True)) + sender_pubkey = btc.add_pubkeys([reusable_donation_pubkey, + btc.privtopub(c+'01', True)], True) + sender_address = btc.pubtoaddr(sender_pubkey, get_p2pk_vbyte()) + log.debug('sending coins to ' + sender_address) + return sender_address, sign_k + +def check_utxo_blacklist(commitment, persist=False): + """Compare a given commitment (H(P2) for PoDLE) + with the persisted blacklist log file; + if it has been used before, return False (disallowed), + else return True. + If flagged, persist the usage of this commitment to the blacklist file. + """ + #TODO format error checking? + fname = "blacklist" + if jm_single().config.get("BLOCKCHAIN", "blockchain_source") == 'regtest': + fname += "_" + jm_single().nickname + with jm_single().blacklist_file_lock: + if os.path.isfile(fname): + with open(fname, "rb") as f: + blacklisted_commitments = [x.strip() for x in f.readlines()] + else: + blacklisted_commitments = [] + if commitment in blacklisted_commitments: + return False + elif persist: + blacklisted_commitments += [commitment] + with open(fname, "wb") as f: + f.write('\n'.join(blacklisted_commitments)) + f.flush() + #If the commitment is new and we are *not* persisting, nothing to do + #(we only add it to the list on sending io_auth, which represents actual + #usage). + return True + def load_program_config(): + #set the location of joinmarket + jmkt_dir = os.path.dirname(os.path.dirname(os.path.realpath(__file__))) + log.debug("Joinmarket directory is: " + str(jmkt_dir)) global_singleton.config.readfp(io.BytesIO(defaultconfig)) - loadedFiles = global_singleton.config.read( - [global_singleton.config_location]) + jmkt_config_location = os.path.join(jmkt_dir, global_singleton.config_location) + loadedFiles = global_singleton.config.read([jmkt_config_location]) # Create default config file if not found if len(loadedFiles) != 1: - with open(global_singleton.config_location, "w") as configfile: + with open(jmkt_config_location, "w") as configfile: configfile.write(defaultconfig) # check for sections for s in required_options: if s not in global_singleton.config.sections(): raise Exception( - "Config file does not contain the required section: " + s) + "Config file does not contain the required section: " + s) # then check for specific options for k, v in required_options.iteritems(): for o in v: @@ -234,22 +309,23 @@ def load_program_config(): try: global_singleton.maker_timeout_sec = global_singleton.config.getint( - 'TIMEOUT', 'maker_timeout_sec') + 'TIMEOUT', 'maker_timeout_sec') except NoOptionError: log.debug('TIMEOUT/maker_timeout_sec not found in .cfg file, ' 'using default value') # configure the interface to the blockchain on startup global_singleton.bc_interface = get_blockchain_interface_instance( - global_singleton.config) - - #print warning if not using libsecp256k1 - if not btc.secp_present: - log.debug("WARNING: You are not using the binding to libsecp256k1. The " - "crypto code in use has poorer performance and security " - "properties. Consider installing the binding with `pip install " - "secp256k1`.") - + global_singleton.config) + #set the location of the commitments file + try: + global_singleton.commit_file_location = global_singleton.config.get( + "POLICY", "commit_file_location") + except NoOptionError: + log.debug("No commitment file location in config, using default " + "location cmttools/commitments.json") + btc.set_commitment_file(os.path.join(jmkt_dir, + global_singleton.commit_file_location)) def get_blockchain_interface_instance(_config): # todo: refactor joinmarket module to get rid of loops @@ -269,7 +345,8 @@ def get_blockchain_interface_instance(_config): rpc = JsonRpc(rpc_host, rpc_port, rpc_user, rpc_password) bc_interface = BitcoinCoreInterface(rpc, network) elif source == 'json-rpc': - bitcoin_cli_cmd = _config.get("BLOCKCHAIN", "bitcoin_cli_cmd").split(' ') + bitcoin_cli_cmd = _config.get("BLOCKCHAIN", + "bitcoin_cli_cmd").split(' ') rpc = CliJsonRpc(bitcoin_cli_cmd, testnet) bc_interface = BitcoinCoreInterface(rpc, network) elif source == 'regtest': diff --git a/joinmarket/irc.py b/joinmarket/irc.py index a6c54739..ee400147 100644 --- a/joinmarket/irc.py +++ b/joinmarket/irc.py @@ -8,8 +8,9 @@ import time import Queue -from joinmarket.configure import jm_single, get_config_irc_channel +from joinmarket.configure import get_config_irc_channel, jm_single from joinmarket.message_channel import MessageChannel, CJPeerError, COMMAND_PREFIX +from joinmarket.message_channel import NICK_MAX_ENCODED from joinmarket.enc_wrapper import encrypt_encode, decode_decrypt from joinmarket.support import get_log, chunks from joinmarket.socks import socksocket, setdefaultproxy, PROXY_TYPE_SOCKS5 @@ -69,7 +70,8 @@ def get_irc_text(line): def get_irc_nick(source): - return source[1:source.find('!')] + full_nick = source[1:source.find('!')] + return full_nick[:NICK_MAX_ENCODED+2] class ThrottleThread(threading.Thread): @@ -194,7 +196,7 @@ def shutdown(self): self.give_up = True # Maker callbacks - def _announce_orders(self, orderlist, nick): + def _announce_orders(self, orderlist): """This publishes orders to the pit and to counterparties. Note that it does *not* use chunking. So, it tries to optimise space usage thusly: @@ -209,11 +211,10 @@ def _announce_orders(self, orderlist, nick): fitting as many list entries as possible onto one line, up to the limit of the IRC parameters (see MAX_PRIVMSG_LEN). - nick=None means announce publically. Theoretically, we - could use chunking for the non-public, but for simplicity - just have one function. + Order announce in private is handled by privmsg/_privmsg + using chunking, no longer using this function. """ - header = 'PRIVMSG ' + (nick if nick else self.channel) + ' :' + header = 'PRIVMSG ' + self.channel + ' :' orderlines = [] for i, order in enumerate(orderlist): orderlines.append(order) @@ -227,12 +228,16 @@ def _announce_orders(self, orderlist, nick): def _pubmsg(self, message): line = "PRIVMSG " + self.channel + " :" + message assert len(line) <= MAX_PRIVMSG_LEN - self.send_raw(line) + ob = False + if any([x in line for x in jm_single().ordername_list]): + ob = True + self.send_raw(line, ob) def _privmsg(self, nick, cmd, message): """Send a privmsg to an irc counterparty, using chunking as appropriate for long messages. """ + ob = True if cmd in jm_single().ordername_list else False header = "PRIVMSG " + nick + " :" max_chunk_len = MAX_PRIVMSG_LEN - len(header) - len(cmd) - 4 # 1 for command prefix 1 for space 2 for trailer @@ -244,14 +249,18 @@ def _privmsg(self, nick, cmd, message): trailer = ' ~' if m == message_chunks[-1] else ' ;' if m == message_chunks[0]: m = COMMAND_PREFIX + cmd + ' ' + m - self.send_raw(header + m + trailer) + self.send_raw(header + m + trailer, ob) - def send_raw(self, line): + def change_nick(self, new_nick): + self.nick = new_nick + self.send_raw('NICK ' + self.nick) + + def send_raw(self, line, ob=False): # Messages are queued and prioritised. # This is an addressing of github #300 if line.startswith("PING") or line.startswith("PONG"): self.pingQ.put(line) - elif "relorder" in line or "absorder" in line: + elif ob: self.obQ.put(line) else: self.throttleQ.put(line) @@ -309,11 +318,12 @@ def __handle_line(self, line): raise IOError('we quit') else: if self.on_nick_leave: - self.on_nick_leave(nick) + self.on_nick_leave(nick, self) elif _chunks[1] == '433': # nick in use - # self.nick = random_nick() - self.nick += '_' # helps keep identity constant if just _ added - self.send_raw('NICK ' + self.nick) + # helps keep identity constant if just _ added + #request new nick on *all* channels via callback + if self.on_nick_change: + self.on_nick_change(self.nick + '_') if self.password: if _chunks[1] == 'CAP': if _chunks[3] != 'ACK': @@ -342,7 +352,7 @@ def __handle_line(self, line): elif _chunks[1] == '376': # end of motd self.built_privmsg = {} if self.on_connect: - self.on_connect() + self.on_connect(self) self.send_raw('JOIN ' + self.channel) self.send_raw( 'MODE ' + self.nick + ' +B') # marks as bots on unreal @@ -351,7 +361,7 @@ def __handle_line(self, line): elif _chunks[1] == '366': # end of names list log.debug('Connected to IRC and joined channel') if self.on_welcome: - self.on_welcome() + self.on_welcome(self) #informs mc-collection that we are ready for use elif _chunks[1] == '332' or _chunks[1] == 'TOPIC': # channel topic topic = get_irc_text(line) self.on_set_topic(topic) @@ -363,40 +373,41 @@ def __handle_line(self, line): raise IOError(fmt(get_irc_nick(_chunks[0]), get_irc_text(line))) else: if self.on_nick_leave: - self.on_nick_leave(target) + self.on_nick_leave(target, self) elif _chunks[1] == 'PART': nick = get_irc_nick(_chunks[0]) if self.on_nick_leave: - self.on_nick_leave(nick) - - # todo: cleanup - # elif _chunks[1] == 'JOIN': - # channel = _chunks[2][1:] - # nick = get_irc_nick(_chunks[0]) - # - # elif chunks[1] == '005': - # self.motd_fd = open("motd.txt", "w") - # elif chunks[1] == '372': - # self.motd_fd.write(get_irc_text(line) + "\n") - # elif chunks[1] == '251': - # self.motd_fd.close() + self.on_nick_leave(nick, self) + elif _chunks[1] == '005': + ''' + :port80b.se.quakenet.org 005 J5BzJGGfyw5GaPc MAXNICKLEN=15 + TOPICLEN=250 AWAYLEN=160 KICKLEN=250 CHANNELLEN=200 + MAXCHANNELLEN=200 CHANTYPES=#& PREFIX=(ov)@+ STATUSMSG=@+ + CHANMODES=b,k,l,imnpstrDducCNMT CASEMAPPING=rfc1459 + NETWORK=QuakeNet :are supported by this server + ''' + for chu in _chunks[3:]: + if chu[0] == ':': + break + if chu.lower().startswith('network='): + self.hostid = chu[8:] + log.debug('found network name: ' + self.hostid + ';') def __init__(self, - given_nick, + configdata, username='username', realname='realname', password=None): MessageChannel.__init__(self) self.give_up = True - self.cjpeer = None # subclasses have to set this to self - self.given_nick = given_nick - self.nick = given_nick - config = jm_single().config - self.serverport = (config.get("MESSAGING", "host"), - int(config.get("MESSAGING", "port"))) - self.socks5_host = config.get("MESSAGING", "socks5_host") - self.socks5_port = int(config.get("MESSAGING", "socks5_port")) - self.channel = get_config_irc_channel() + self.serverport = (configdata['host'], configdata['port']) + #default hostid for use with miniircd which doesnt send NETWORK + self.hostid = configdata['host'] + str(configdata['port']) + self.socks5 = configdata["socks5"] + self.usessl = configdata["usessl"] + self.socks5_host = configdata["socks5_host"] + self.socks5_port = int(configdata["socks5_port"]) + self.channel = get_config_irc_channel(configdata["channel"]) self.userrealname = (username, realname) if password and len(password) == 0: password = None @@ -415,9 +426,8 @@ def run(self): while not self.give_up: try: - config = jm_single().config log.debug('connecting') - if config.get("MESSAGING", "socks5").lower() == 'true': + if self.socks5.lower() == 'true': log.debug("Using socks5 proxy %s:%d" % (self.socks5_host, self.socks5_port)) setdefaultproxy(PROXY_TYPE_SOCKS5, @@ -428,7 +438,7 @@ def run(self): self.sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) self.sock.connect(self.serverport) - if config.get("MESSAGING", "usessl").lower() == 'true': + if self.usessl.lower() == 'true': self.sock = ssl.wrap_socket(self.sock) self.fd = self.sock.makefile() self.password = None @@ -460,7 +470,7 @@ def run(self): except Exception as e: pass if self.on_disconnect: - self.on_disconnect() + self.on_disconnect(self) log.debug('disconnected irc') if not self.give_up: time.sleep(30) diff --git a/joinmarket/maker.py b/joinmarket/maker.py index 2b57e7d6..2d309262 100644 --- a/joinmarket/maker.py +++ b/joinmarket/maker.py @@ -8,12 +8,13 @@ import bitcoin as btc from joinmarket import IRCMessageChannel -from joinmarket.configure import get_p2pk_vbyte, load_program_config, jm_single +from joinmarket.configure import get_p2pk_vbyte, load_program_config, jm_single, \ + check_utxo_blacklist from joinmarket.enc_wrapper import init_keypair, as_init_encryption, init_pubkey, \ NaclError from joinmarket.support import get_log, calc_cj_fee, debug_dump_object -from joinmarket.taker import CoinJoinerPeer +from joinmarket.taker import OrderbookWatch from joinmarket.wallet import Wallet log = get_log() @@ -89,21 +90,60 @@ def populate_utxo_data(): # orders to find out which addresses you use self.maker.msgchan.send_pubkey(nick, self.kp.hex_pk()) - def auth_counterparty(self, nick, i_utxo_pubkey, btc_sig): - self.i_utxo_pubkey = i_utxo_pubkey - - if not btc.ecdsa_verify(self.taker_pk, btc_sig, self.i_utxo_pubkey): - print('signature didnt match pubkey and message') + def auth_counterparty(self, nick, cr): + #deserialize the commitment revelation + cr_dict = btc.PoDLE.deserialize_revelation(cr) + #check the validity of the proof of discrete log equivalence + tries = jm_single().config.getint("POLICY", "taker_utxo_retries") + def reject(msg): + log.debug("Counterparty commitment not accepted, reason: " + msg) return False + if not btc.verify_podle(cr_dict['P'], cr_dict['P2'], cr_dict['sig'], + cr_dict['e'], self.maker.commit, + index_range=range(tries)): + reason = "verify_podle failed" + return reject(reason) + #finally, check that the proffered utxo is real, old enough, large enough, + #and corresponds to the pubkey + res = jm_single().bc_interface.query_utxo_set([cr_dict['utxo']], + includeconf=True) + if len(res) != 1 or not res[0]: + reason = "authorizing utxo is not valid" + return reject(reason) + age = jm_single().config.getint("POLICY", "taker_utxo_age") + if res[0]['confirms'] < age: + reason = "commitment utxo not old enough: " + str(res[0]['confirms']) + return reject(reason) + reqd_amt = int(self.cj_amount * jm_single().config.getint( + "POLICY", "taker_utxo_amtpercent") / 100.0) + if res[0]['value'] < reqd_amt: + reason = "commitment utxo too small: " + str(res[0]['value']) + return reject(reason) + if res[0]['address'] != btc.pubkey_to_address(cr_dict['P'], + get_p2pk_vbyte()): + reason = "Invalid podle pubkey: " + str(cr_dict['P']) + return reject(reason) + # authorisation of taker passed - # (but input utxo pubkey is checked in verify_unsigned_tx). + # Send auth request to taker - # TODO the next 2 lines are a little inefficient. - btc_key = self.maker.wallet.get_key_from_addr(self.cj_addr) - btc_pub = btc.privtopub(btc_key) - btc_sig = btc.ecdsa_sign(self.kp.hex_pk(), btc_key) - self.maker.msgchan.send_ioauth(nick, self.utxos.keys(), btc_pub, - self.change_addr, btc_sig) + # Need to choose an input utxo pubkey to sign with + # (no longer using the coinjoin pubkey from 0.2.0) + # Just choose the first utxo in self.utxos and retrieve key from wallet. + auth_address = self.utxos[self.utxos.keys()[0]]['address'] + auth_key = self.maker.wallet.get_key_from_addr(auth_address) + auth_pub = btc.privtopub(auth_key) + btc_sig = btc.ecdsa_sign(self.kp.hex_pk(), auth_key) + self.maker.msgchan.send_ioauth(nick, self.utxos.keys(), auth_pub, + self.cj_addr, self.change_addr, btc_sig) + #In case of *blacklisted (ie already used) commitments, we already + #broadcasted them on receipt; in case of valid, and now used commitments, + #we broadcast them here, and not early - to avoid accidentally + #blacklisting commitments that are broadcast between makers in real time + #for the same transaction. + self.maker.transfer_commitment(self.maker.commit) + #now persist the fact that the commitment is actually used. + check_utxo_blacklist(self.maker.commit, persist=True) return True def recv_tx(self, nick, txhex): @@ -133,7 +173,6 @@ def recv_tx(self, nick, txhex): jm_single().bc_interface.add_tx_notify( self.tx, self.unconfirm_callback, self.confirm_callback, self.cj_addr) - log.debug('sending sigs ' + str(sigs)) self.maker.msgchan.send_sigs(nick, sigs) self.maker.active_orders[nick] = None @@ -164,16 +203,6 @@ def confirm_callback(self, txd, txid, confirmations): def verify_unsigned_tx(self, txd): tx_utxo_set = set(ins['outpoint']['hash'] + ':' + str( ins['outpoint']['index']) for ins in txd['ins']) - # complete authentication: check the tx input uses the authing pubkey - input_utxo_data = jm_single().bc_interface.query_utxo_set( - list(tx_utxo_set)) - - if None in input_utxo_data: - return False, 'some utxos already spent or not confirmed yet' - input_addresses = [u['address'] for u in input_utxo_data] - if btc.pubtoaddr( - self.i_utxo_pubkey, get_p2pk_vbyte()) not in input_addresses: - return False, "authenticating bitcoin address is not contained" my_utxo_set = set(self.utxos.keys()) if not tx_utxo_set.issuperset(my_utxo_set): @@ -214,16 +243,18 @@ class CJMakerOrderError(StandardError): pass -class Maker(CoinJoinerPeer): +class Maker(OrderbookWatch): def __init__(self, msgchan, wallet): - CoinJoinerPeer.__init__(self, msgchan) + OrderbookWatch.__init__(self, msgchan) self.msgchan.register_channel_callbacks(self.on_welcome, self.on_set_topic, None, None, self.on_nick_leave, None) msgchan.register_maker_callbacks(self.on_orderbook_requested, self.on_order_fill, self.on_seen_auth, - self.on_seen_tx, self.on_push_tx) - msgchan.cjpeer = self + self.on_seen_tx, self.on_push_tx, + self.on_commitment_seen, + self.on_commitment_transferred) + msgchan.set_cjpeer(self) self.active_orders = {} self.wallet = wallet @@ -237,16 +268,77 @@ def get_crypto_box_from_nick(self, nick): 'wrong ordering of protocol events, no crypto object, nick=' + nick) return None + elif not self.active_orders[nick]: + return None else: return self.active_orders[nick].crypto_box - def on_orderbook_requested(self, nick): - self.msgchan.announce_orders(self.orderlist, nick) + def on_orderbook_requested(self, nick, mc=None): + self.msgchan.announce_orders(self.orderlist, nick, mc) + + def on_commitment_transferred(self, nick, commitment): + """Triggered when a privmsg is received from another maker + with a commitment to announce in public (obfuscation of source). + We simply post it in public (not affected by whether we ourselves + are *accepting* commitment broadcasts. + """ + self.msgchan.pubmsg("!hp2 " + commitment) + + def on_commitment_seen(self, nick, commitment): + """Triggered when we see a commitment for blacklisting + appear in the public pit channel. If the policy is set, + we blacklist this commitment. + """ + if jm_single().config.has_option("POLICY", "accept_commitment_broadcasts"): + blacklist_add = jm_single().config.getint("POLICY", + "accept_commitment_broadcasts") + else: + blacklist_add = 0 + if blacklist_add > 0: + #just add if necessary, ignore return value. + check_utxo_blacklist(commitment, persist=True) + log.debug("Received commitment broadcast by other maker: " + str( + commitment) + ", now blacklisted.") + else: + log.debug("Received commitment broadcast by other maker: " + str( + commitment) + ", ignored.") + + def transfer_commitment(self, commit): + """Send this commitment via privmsg to one (random) + other maker. + """ + crow = self.db.execute( + 'SELECT DISTINCT counterparty FROM orderbook ORDER BY ' + + 'RANDOM() LIMIT 1;' + ).fetchone() + if crow is None: + return + counterparty = crow['counterparty'] + #TODO de-hardcode hp2 + log.debug("Sending commitment to: " + str(counterparty)) + self.msgchan.privmsg(counterparty, 'hp2', commit) - def on_order_fill(self, nick, oid, amount, taker_pubkey): + def on_order_fill(self, nick, oid, amount, taker_pubkey, commit): if nick in self.active_orders and self.active_orders[nick] is not None: self.active_orders[nick] = None log.debug('had a partially filled order but starting over now') + if not commit[0] == "P": + self.msgchan.send_error( + nick, "Unsupported commitment type: " + str(commit[0])) + return + #Strip the type byte before processing + scommit = commit[1:] + if not check_utxo_blacklist(scommit): + log.debug("Taker utxo commitment is blacklisted, rejecting.") + self.msgchan.send_error(nick, + "Commitment is blacklisted: " + str(scommit)) + #Note that broadcast is happening here to reflect an already + #consumed commitment; it can also be broadcast separately (earlier) on + #valid usage in CoinjoinOrder.auth_counterparty(). + #Keep the type byte for communication so not scommit: + self.transfer_commitment(commit) + return + self.commit = scommit self.wallet_unspent_lock.acquire() try: self.active_orders[nick] = CoinJoinOrder(self, nick, oid, amount, @@ -254,12 +346,12 @@ def on_order_fill(self, nick, oid, amount, taker_pubkey): finally: self.wallet_unspent_lock.release() - def on_seen_auth(self, nick, pubkey, sig): + def on_seen_auth(self, nick, cr): if nick not in self.active_orders or self.active_orders[nick] is None: self.msgchan.send_error(nick, 'No open order from this nick') - self.active_orders[nick].auth_counterparty(nick, pubkey, sig) - # TODO if auth_counterparty returns false, remove this order from active_orders - # and send an error + if not self.active_orders[nick].auth_counterparty(nick, cr): + self.active_orders[nick] = None + self.msgchan.send_error(nick, "Authorisation failed") def on_seen_tx(self, nick, txhex): if nick not in self.active_orders or self.active_orders[nick] is None: @@ -323,7 +415,7 @@ def create_my_orders(self): for utxo, addrvalue in self.wallet.unspent.iteritems(): total_value += addrvalue['value'] - order = {'oid': 0, 'ordertype': 'relorder', 'minsize': 0, + order = {'oid': 0, 'ordertype': 'reloffer', 'minsize': 0, 'maxsize': total_value, 'txfee': 10000, 'cjfee': '0.002'} return [order] """ @@ -332,7 +424,7 @@ def create_my_orders(self): orderlist = [] for utxo, addrvalue in self.wallet.unspent.iteritems(): order = {'oid': self.get_next_oid(), - 'ordertype': 'absorder', + 'ordertype': 'absoffer', 'minsize': 12000, 'maxsize': addrvalue['value'], 'txfee': 10000, @@ -383,7 +475,7 @@ def on_tx_confirmed(self, cjorder, confirmations, txid): addr = btc.script_to_address(out['script'], get_p2pk_vbyte()) if addr == cjorder.change_addr: neworder = {'oid': self.get_next_oid(), - 'ordertype': 'absorder', + 'ordertype': 'absoffer', 'minsize': 12000, 'maxsize': out['value'], 'txfee': 10000, @@ -392,7 +484,7 @@ def on_tx_confirmed(self, cjorder, confirmations, txid): to_announce.append(neworder) if addr == cjorder.cj_addr: neworder = {'oid': self.get_next_oid(), - 'ordertype': 'absorder', + 'ordertype': 'absoffer', 'minsize': 12000, 'maxsize': out['value'], 'txfee': 10000, diff --git a/joinmarket/message_channel.py b/joinmarket/message_channel.py index a654ef8e..ac741e9f 100644 --- a/joinmarket/message_channel.py +++ b/joinmarket/message_channel.py @@ -1,13 +1,19 @@ -import base64, abc +import base64, abc, threading, time, hashlib, os, binascii from joinmarket.enc_wrapper import encrypt_encode, decode_decrypt from joinmarket.support import get_log, chunks from joinmarket.configure import jm_single +import bitcoin as btc +from functools import wraps COMMAND_PREFIX = '!' +JOINMARKET_NICK_HEADER = 'J' +NICK_HASH_LENGTH = 10 +NICK_MAX_ENCODED = 14 #comes from base58 expansion; recalculate if above changes encrypted_commands = ["auth", "ioauth", "tx", "sig"] -plaintext_commands = ["fill", "error", "pubkey", "orderbook", "relorder", - "absorder", "push"] +plaintext_commands = ["fill", "error", "pubkey", "orderbook", "push"] +plaintext_commands += jm_single().ordername_list +plaintext_commands += jm_single().commitment_broadcast_list log = get_log() @@ -15,6 +21,574 @@ class CJPeerError(StandardError): pass +class MChannelThread(threading.Thread): + def __init__(self, mc): + threading.Thread.__init__(self, name='MCThread') + self.daemon = True + self.mc = mc + + def run(self): + self.mc.run() + +class MessageChannelCollection(object): + """Class which encapsulates a set of + message channels. Maintains state about active + connections to counterparties, and state of + encapsulated message channel instances. + Public messages are broadcast over all available + channels, while privmsgs with one counterparty are + "locked" to the channel on which they are initiated, + although bear in mind they need not be the same for + both sides of the conversation. + In the current joinmarket protocol, this "lock" + is set at the time of the !reloffer (etc) privmsg or + pubmsg from the maker. + Note that MessageChannel implementations must support + asynchronous messaging (adding to Queue.Queue objects, + which are thread safe, e.g.) + Callback chain is in some cases extended with an extra + layer, e.g. to manage a "connected" state across all + encapsulated message channels. + """ + + def check_privmsg(func): + """decorator to check if private messages + are correctly activated + """ + @wraps(func) + def func_wrapper(inst, *args, **kwargs): + cp = args[0] + if cp in inst.active_channels: + return func(inst, *args, **kwargs) + else: + for mc in inst.available_channels(): + #nicks_seen[mc] guaranteed to exist + #from constructor + if cp in inst.nicks_seen[mc]: + log.debug("Dynamic switch nick: " + cp) + inst.active_channels[cp] = mc + #early return on first success; + #means that we assume that if we have + #ever seen a message from this counterparty + #on one messagechannel which is currently active, + #we assume it's still + #available. Of course, this is optimistic, + #but still much better to do this than to + #immediately give up when any one connection + #is broken. + return func(inst, *args, **kwargs) + #Failure to send is a critical error for a transaction, + #but should not kill the bot. So, we don't raise an + #exception, but rather allow sending to continue, which + #should usually result in tx completion just timing out. + log.debug("Couldn't find a route to send privmsg") + log.debug("For counterparty: " + str(cp)) + return func_wrapper + + def __init__(self, mchannels): + self.mchannels = mchannels + #To keep track of chosen channels + #for private messaging counterparties. + self.active_channels = {} + #To keep track of message channel status; + #0: not started 1: started 2: failed/broken/inactive + self.mc_status = dict([(x, 0) for x in self.mchannels]) + #To keep track of counterparties having at least once + #made their presence known on a channel + self.nicks_seen = {} + for mc in self.mchannels: + self.nicks_seen[mc] = set() + #callback to mark nicks as seen when they privmsg + mc.on_privmsg_trigger = self.on_privmsg + #keep track of whether we want to deliberately + #shut down the connections + self.give_up = False + #only allow on_welcome() to fire once. + self.welcomed = False + #control access + self.mc_lock = threading.Lock() + #Create an ephemeral keypair for the duration + #of this run, same across all message channels, + #and set the nickname for all message channels using it. + self.nick_priv = hashlib.sha256(os.urandom(16)).hexdigest() + '01' + self.nick_pubkey = btc.privtopub(self.nick_priv) + self.nick_pkh_raw = hashlib.sha256(self.nick_pubkey).digest()[ + :NICK_HASH_LENGTH] + self.nick_pkh = btc.changebase(self.nick_pkh_raw, 256, 58) + #right pad to maximum possible; b58 is not fixed length. + #Use 'O' as one of the 4 not included chars in base58. + self.nick_pkh += 'O' * (NICK_MAX_ENCODED - len(self.nick_pkh)) + #The constructed length will be 1 + 1 + NICK_MAX_ENCODED + self.nick = JOINMARKET_NICK_HEADER + str( + jm_single().JM_VERSION) + self.nick_pkh + jm_single().nickname = self.nick + for mc in self.mchannels: + mc.set_nick(self.nick, self.nick_priv, self.nick_pubkey) + + def available_channels(self): + return [x for x in self.mchannels if self.mc_status[x]==1] + + def unavailable_channels(self): + return [x for x in self.mchannels if self.mc_status[x] != 1] + + def flush_nicks(self): + """Any message channel which is not + active must wipe any state information on peers + connected for that message channel. If a peer is + available on another chan, switch the active_channel + for that nick to (an)(the) other, to make failure + to communicate as unlikely as possible. + """ + for mc in self.unavailable_channels(): + self.nicks_seen[mc] = set() + ac = self.active_channels + for peer in [x for x in ac if ac[x] == mc]: + for mc2 in self.available_channels(): + if peer in self.nicks_seen[mc2]: + log.debug("Dynamically switching: " + peer + " to: " + \ + str(mc2.serverport)) + self.active_channels[peer] = mc2 + break + #Remove all entries for the newly unavailable channel + self.active_channels = dict([(a, ac[a]) for a in ac if ac[a] != mc]) + + def set_cjpeer(self, cjpeer): + for mc in self.mchannels: + mc.cjpeer = cjpeer + + def add_channel(self, mchannel): + """TODO Not currently in use, + may be some issues with intialization. + """ + if mchannel not in self.mchannels: + self.mc_status[mc] = 0 + self.nicks_seen[mc] = set() + self.mchannels += mchannel + self.mchannels = list(set(self.mchannels)) + + def see_nick(self, nick, mc): + with self.mc_lock: + self.nicks_seen[mc].add(nick) + + def unsee_nick(self, nick, mc): + with self.mc_lock: + self.nicks_seen[mc] = self.nicks_seen[mc].difference(set([nick])) + + def run(self, failures=None): + """At the moment this is effectively a + do-nothing main loop. May be suboptimal. + For now it allows us to receive the + shutdown() signal for all message channels + and propagate it. + Additionally, for testing, a parameter 'failures' + may be passed, a tuple (type, message channel index, count) + which will perform a connection shutdown of type type + after iteration count count on message channel + self.mchannels[channel index]. + """ + for mc in self.mchannels: + MChannelThread(mc).start() + i = 0 + while True: + time.sleep(1) + i += 1 + if self.give_up: + log.debug("Shutting down all connections") + break + #feature only used for testing: + #deliberately shutdown a connection at a certain time. + #TODO may not be sufficiently deterministic. + if failures and i==failures[2]: + if failures[0] == 'break': + self.mchannels[failures[1]].close() + elif failures[0] == 'shutdown': + self.mchannels[failures[1]].shutdown() + else: + raise NotImplementedError("Failure injection type unknown") + + #UNCONDITIONAL PUBLIC/BROADCAST: use all message + #channels for these functions. + + def shutdown(self): + """Stop the main loop of the message channel, + shutting down subsidiary resources gracefully. + Note that unexpected disconnections MUST be + handled by the implementation itself (restarting + as appropriate). + """ + for mc in self.available_channels(): + mc.shutdown() + self.give_up = True + + def pubmsg(self, msg): + """Send a message onto the shared, public + channels (the joinmarket pit). + """ + for mc in self.available_channels(): + mc.pubmsg(msg) + + def cancel_orders(self, oid_list): + for mc in self.available_channels(): + mc.cancel_orders(oid_list) + + # OrderbookWatch callback + def request_orderbook(self): + for mc in self.available_channels(): + mc.request_orderbook() + + #END PUBLIC/BROADCAST SECTION + + def privmsg(self, nick, cmd, message, mc=None): + """Send a message to a specific counterparty, + either specifying a single message channel, or + allowing it to be deduced from self.active_channels dict + """ + if mc is not None: + if mc not in self.available_channels(): + #raise because implies logic error + raise Exception( + "Tried to privmsg on an unavailable message channel.") + else: + mc.privmsg(nick, cmd, message) + return + if nick in self.active_channels: + self.active_channels[nick].privmsg(nick, cmd, message) + return + else: + log.debug("Failed to send message to: " + str(nick) + \ + "; cannot find on any message channel.") + return + + def announce_orders(self, orderlist, nick=None, new_mc=None): + """Send orders defined in list orderlist either + to the shared public channel (pit), on all + message channels, if nick=None, + or to an individual counterparty nick, as + privmsg, on a specific mc. + """ + order_keys = ['oid', 'minsize', 'maxsize', 'txfee', 'cjfee'] + orderlines = [] + for order in orderlist: + orderlines.append(COMMAND_PREFIX + order['ordertype'] + \ + ' ' + ' '.join([str(order[k]) for k in order_keys])) + if new_mc is not None and new_mc not in self.available_channels(): + log.debug( + "Tried to announce orders on an unavailable message channel.") + return + if nick is None: + for mc in self.available_channels(): + mc.announce_orders(orderlines) + else: + #we are sending to one cp, so privmsg + #in order to use privmsg, we must set "cmd" to be the first command + #in the first orderline, and the rest are treated like a message. + cmd = orderlist[0]['ordertype'] + msg = ' '.join(orderlines[0].split(' ')[1:]) + msg += ''.join(orderlines[1:]) + if new_mc: + self.privmsg(nick, cmd, msg, new_mc) + else: + for mc in self.available_channels(): + if nick in self.nicks_seen[mc]: + self.privmsg(nick, cmd, msg, mc) + + @check_privmsg + def send_pubkey(self, nick, pubkey): + self.active_channels[nick].privmsg(nick, 'pubkey', pubkey) + + @check_privmsg + def send_ioauth(self, nick, utxo_list, auth_pub, cj_addr, change_addr, sig): + self.active_channels[nick].send_ioauth(nick, utxo_list, auth_pub, + cj_addr, change_addr, sig) + + @check_privmsg + def send_sigs(self, nick, sig_list): + self.active_channels[nick].send_sigs(nick, sig_list) + + # Taker callbacks + def fill_orders(self, nick_order_dict, cj_amount, taker_pubkey, + commitment): + """ + The orders dict does not contain information + about which message channel the counterparty bots are active + on; this can be hacked-around by including that information + in the order data, but this is highly undesirable, partly + architecturally (the joinmarket business logic has no business + knowing about the message channel), and partly because it + would break backwards compatibility. + So, we use a trigger in on_order_seen and assume that it + makes sense to set the active_channel for that nick to the one + it was last seen active on. + """ + for mc in self.available_channels(): + filtered_nick_order_dict = {k:v for k,v in nick_order_dict.iteritems( + ) if mc == self.active_channels[k]} + mc.fill_orders(filtered_nick_order_dict, cj_amount, taker_pubkey, + commitment) + + @check_privmsg + def send_auth(self, nick, cr): + self.active_channels[nick].send_auth(nick, cr) + + @check_privmsg + def send_error(self, nick, errormsg): + #TODO this might need to support non-active nicks TODO + self.active_channels[nick].send_error(nick, errormsg) + + @check_privmsg + def push_tx(self, nick, txhex): + #TODO supporting sending to arbitrary nicks + #adds quite a bit of complexity, not supported + #initially; will fail if nick is not part of TX + self.active_channels[nick].push_tx(nick, txhex) + + def send_tx(self, nick_list, txhex): + """Push out the transaction to nicks + in groups by their message channel. + """ + tx_nick_sets = {} + for nick in nick_list: + if nick not in self.active_channels: + #This could be a fatal error for a transaction, + #but might not be for the bot (tx recreation etc.) + #TODO look for another channel via nicks_seen. + #Rare case so not a high priority. + log.debug( + "Cannot send transaction to nick, not active: " + nick) + return + if self.active_channels[nick] not in tx_nick_sets: + tx_nick_sets[self.active_channels[nick]] = [nick] + else: + tx_nick_sets[self.active_channels[nick]].append(nick) + for mc, nl in tx_nick_sets.iteritems(): + mc.send_tx(nl, txhex) + + #CALLBACKS REGISTRATION SECTION + + # taker commands + def register_taker_callbacks(self, + on_error=None, + on_pubkey=None, + on_ioauth=None, + on_sig=None): + for mc in self.mchannels: + mc.register_taker_callbacks(on_error, + on_pubkey, + on_ioauth, + on_sig) + + def on_connect_trigger(self, mc): + """Mark the specified message channel + as (re) connected. + """ + self.mc_status[mc] = 1 + + def on_disconnect_trigger(self, mc): + """Mark the specified message channel as + disconnected. Track loss of private connections + to individual nicks. If no message channels are + now connected, fire on_disconnect to calling code. + """ + self.mc_status[mc] = 2 + self.flush_nicks() + log.debug("On disconnect fired, nicks_seen is now: " + str(self.nicks_seen)) + if not any([x==1 for x in self.mc_status.values()]): + if self.on_disconnect: + self.on_disconnect() + + def on_welcome_trigger(self, mc): + """Update status of specified message channel + as connected. If all required message channels + are initialized (not state 0), fire the + on_welcome() event to calling code to signal + that processing can start. + This is wrapped with a lock as can be fired by + message channel child threads. + """ + with self.mc_lock: + if self.welcomed: + return + #This trigger indicates successful login + #so we update status. + self.mc_status[mc] = 1 + #This way broadcasts orders or requests ONCE to ALL mchans + #which are actually available. + if not any([x == 0 for x in self.mc_status.values()]): + if self.on_welcome: + self.on_welcome() + self.welcomed = True + + def on_nick_leave_trigger(self, nick, mc): + """If a nick leaves one message channel, + and we are currently talking to it on that + channel, attempt to dynamically switch to + another channel on which it has been seen. + If we are currently talking to it on a different + channel, we ignore the signal, since it shouldn't + interrupt processing. + If we are not currently talking to it at all, + just call on_nick_leave (which currently does nothing). + """ + + #mark the nick as 'unseen' on that channel + self.unsee_nick(nick, mc) + if nick not in self.active_channels: + if self.on_nick_leave: + self.on_nick_leave(nick) + elif self.active_channels[nick] == mc: + del self.active_channels[nick] + #Attempt to dynamically switch channels + #Is the nick available on another channel? + other_channels = [x for x in self.available_channels() if x != mc] + if len(other_channels) == 0: + log.debug( + "Cannot reconnect to dropped nick, no connections available.") + if self.on_nick_leave: + self.on_nick_leave(nick) + return + for oc in other_channels: + if nick in self.nicks_seen[oc]: + log.debug("Found a new channel, setting to: " + nick + \ + "," + str(oc.serverport)) + self.active_channels[nick] = oc + #Note we don't call on_nick_leave in this case + return + #If loop completed without success, we failed to find + #this counterparty anywhere else + log.debug("Nick: " + nick + " has left.") + if self.on_nick_leave: + self.on_nick_leave(nick) + #The remaining case is if the channel that the + #nick has left is not the one we're currently using. + return + + def register_channel_callbacks(self, + on_welcome=None, + on_set_topic=None, + on_connect=None, + on_disconnect=None, + on_nick_leave=None, + on_nick_change=None): + """Special cases: + on_welcome: we maintain it + in this class, since we only want to trigger arrival + when all channels are joined, not multiple times, then + broadcast whatever it is we want to broadcast on arrival. + + on_nick_leave: this needs to be maintained in this class, + since a nick only leaves the pit when it has departed *all* our + message channels. + + on_nick_change: a bot which changes its nick on one channel + must also successfully change its nick on all channels, or quit. + + on_disconnect: must be maintained here; if a bot disconnects + only one it must remain viable, otherwise this has no point! + + on_connect: must reset the message channel status to connected. + """ + self.on_welcome = on_welcome + self.on_disconnect = on_disconnect + self.on_nick_leave = on_nick_leave + self.on_connect = on_connect + self.on_nick_change = on_nick_change + for mc in self.mchannels: + mc.register_channel_callbacks(self.on_welcome_trigger, + on_set_topic, + self.on_connect_trigger, + self.on_disconnect_trigger, + self.on_nick_leave_trigger, + self.on_nick_change_trigger, + self.see_nick) + + def on_nick_change_trigger(self, new_nick): + """If any underlying messagechannel object fails to register + a nick/username, trigger all of them to change to the newly + chosen nick/user. + """ + for mc in self.available_channels(): + mc.change_nick(new_nick) + if self.on_nick_change: + self.on_nick_change(new_nick) + + def on_order_seen_trigger(self, mc, counterparty, oid, ordertype, minsize, + maxsize, txfee, cjfee): + """This is the entry point into private messaging. + Hence, it fixes for the rest of the conversation, which + message channel the bots are going to communicate over + (privately). + Use the orderbook update as a signal that this counterparty (nick) + is present on this message channel, before passing to calling code. + Note that this will get called at least once per message channel, + so it will simply end up setting the active channel to the last one + that arrives. + """ + #Note that the counterparty will be added to the set for *each* + #message channel where it has published an order (priv or pub), + #so that we can hope to contact it at any one of those mcs. + self.nicks_seen[mc].add(counterparty) + + self.active_channels[counterparty] = mc + if self.on_order_seen: + self.on_order_seen(counterparty, oid, ordertype, minsize, maxsize, + txfee, cjfee) + + # orderbook watcher commands + def register_orderbookwatch_callbacks(self, + on_order_seen=None, + on_order_cancel=None): + """Special cases: + on_order_seen: use it as a trigger for presence of nick. + on_order_cancel: what happens if cancel/modify in one place + but not another? TODO + """ + self.on_order_seen = on_order_seen + for mc in self.mchannels: + mc.register_orderbookwatch_callbacks(self.on_order_seen_trigger, + on_order_cancel) + + def on_orderbook_requested_trigger(self, nick, mc): + """Update nicks_seen state to reflect presence of + taker on this message channel before pass-through. + """ + self.see_nick(nick, mc) + if self.on_orderbook_requested: + self.on_orderbook_requested(nick, mc) + + # maker commands + def register_maker_callbacks(self, + on_orderbook_requested=None, + on_order_fill=None, + on_seen_auth=None, + on_seen_tx=None, + on_push_tx=None, + on_commitment_seen=None, + on_commitment_transferred=None): + """Special cases: + on_orderbook_requested must trigger addition to the nicks_seen + database, so that makers can know that a taker is in principle + available on this message channel. + """ + self.on_orderbook_requested = on_orderbook_requested + for mc in self.mchannels: + mc.register_maker_callbacks(self.on_orderbook_requested_trigger, + on_order_fill, + on_seen_auth, + on_seen_tx, + on_push_tx, + on_commitment_seen, + on_commitment_transferred) + + def on_privmsg(self, nick, mchan): + """Registered as a callback for all mchannels: + set the nick as seen on privmsg, as it may not + be triggered if it doesn't issue a pubmsg. + """ + if mchan in self.available_channels(): + self.see_nick(nick, mchan) + #Should not be reached; but in weird case that the channel + #is not available, there is nothing to do. + class MessageChannel(object): __metaclass__ = abc.ABCMeta """Abstract class which implements a way for bots to communicate. @@ -31,6 +605,8 @@ def __init__(self): self.on_disconnect = None self.on_nick_leave = None self.on_nick_change = None + self.on_pubmsg_trigger = None + self.on_privmsg_trigger = None # orderbook watch functions self.on_order_seen = None self.on_order_cancel = None @@ -46,6 +622,7 @@ def __init__(self): self.on_seen_tx = None self.on_push_tx = None + self.cjpeer = None """THIS SECTION MUST BE IMPLEMENTED BY SUBCLASSES""" #In addition to the below functions, the implementation @@ -83,23 +660,36 @@ def _announce_orders(self, orderlist, nick): or to an individual counterparty nick. Note that calling code will access this via self.announce_orders.""" + @abc.abstractmethod + def change_nick(self, new_nick): + """Change the nick/username for this message channel + instance to new_nick + """ + """END OF SUBCLASS IMPLEMENTATION SECTION""" - # callbacks for everyone - # some of these many not have meaning in a future channel, like bitmessage + def set_nick(self, nick, nick_priv, nick_pubkey): + self.given_nick = nick + self.nick = self.given_nick + self.nick_priv = nick_priv + self.nick_pubkey = nick_pubkey + def register_channel_callbacks(self, on_welcome=None, on_set_topic=None, on_connect=None, on_disconnect=None, on_nick_leave=None, - on_nick_change=None): + on_nick_change=None, + on_pubmsg_trigger=None): self.on_welcome = on_welcome self.on_set_topic = on_set_topic self.on_connect = on_connect self.on_disconnect = on_disconnect self.on_nick_leave = on_nick_leave self.on_nick_change = on_nick_change + #Fire to MCcollection to mark nicks as "seen" + self.on_pubmsg_trigger = on_pubmsg_trigger # orderbook watcher commands def register_orderbookwatch_callbacks(self, @@ -125,20 +715,19 @@ def register_maker_callbacks(self, on_order_fill=None, on_seen_auth=None, on_seen_tx=None, - on_push_tx=None): + on_push_tx=None, + on_commitment_seen=None, + on_commitment_transferred=None): self.on_orderbook_requested = on_orderbook_requested self.on_order_fill = on_order_fill self.on_seen_auth = on_seen_auth self.on_seen_tx = on_seen_tx self.on_push_tx = on_push_tx + self.on_commitment_seen = on_commitment_seen + self.on_commitment_transferred = on_commitment_transferred - def announce_orders(self, orderlist, nick=None): - order_keys = ['oid', 'minsize', 'maxsize', 'txfee', 'cjfee'] - orderlines = [] - for order in orderlist: - orderlines.append(COMMAND_PREFIX + order['ordertype'] + \ - ' ' + ' '.join([str(order[k]) for k in order_keys])) - self._announce_orders(orderlines, nick) + def announce_orders(self, orderlines): + self._announce_orders(orderlines) def check_for_orders(self, nick, _chunks): if _chunks[0] in jm_single().ordername_list: @@ -151,7 +740,7 @@ def check_for_orders(self, nick, _chunks): txfee = _chunks[4] cjfee = _chunks[5] if self.on_order_seen: - self.on_order_seen(counterparty, oid, ordertype, minsize, + self.on_order_seen(self, counterparty, oid, ordertype, minsize, maxsize, txfee, cjfee) except IndexError as e: log.warning(e) @@ -161,6 +750,30 @@ def check_for_orders(self, nick, _chunks): finally: return True return False + + def check_for_commitments(self, nick, _chunks, private=False): + """If a commitment message is found in a pubmsg, trigger + callback on_commitment_seen, if as a privmsg, trigger + callback on_commitment_transferred. These callbacks are (for now) + only used by Makers. + """ + if _chunks[0] in jm_single().commitment_broadcast_list: + try: + counterparty = nick + commitment = _chunks[1] + if private: + if self.on_commitment_transferred: + self.on_commitment_transferred(counterparty, commitment) + else: + if self.on_commitment_seen: + self.on_commitment_seen(counterparty, commitment) + except IndexError as e: + log.warning(e) + log.debug('index error parsing chunks, possibly malformed' + 'offer by other party. No user action required.') + finally: + return True + return False def cancel_orders(self, oid_list): clines = [COMMAND_PREFIX + 'cancel ' + str(oid) for oid in oid_list] @@ -169,9 +782,9 @@ def cancel_orders(self, oid_list): def send_pubkey(self, nick, pubkey): self.privmsg(nick, 'pubkey', pubkey) - def send_ioauth(self, nick, utxo_list, cj_pubkey, change_addr, sig): - authmsg = (str(','.join(utxo_list)) + ' ' + cj_pubkey + ' ' + - change_addr + ' ' + sig) + def send_ioauth(self, nick, utxo_list, auth_pub, cj_addr, change_addr, sig): + authmsg = str(','.join(utxo_list)) + ' ' + ' '.join([auth_pub, + cj_addr, change_addr, sig]) self.privmsg(nick, 'ioauth', authmsg) def send_sigs(self, nick, sig_list): @@ -184,14 +797,14 @@ def request_orderbook(self): self.pubmsg(COMMAND_PREFIX + 'orderbook') # Taker callbacks - def fill_orders(self, nick_order_dict, cj_amount, taker_pubkey): + def fill_orders(self, nick_order_dict, cj_amount, taker_pubkey, commitment): for c, order in nick_order_dict.iteritems(): msg = str(order['oid']) + ' ' + str(cj_amount) + ' ' + taker_pubkey + msg += ' ' + commitment self.privmsg(c, 'fill', msg) - def send_auth(self, nick, pubkey, sig): - message = pubkey + ' ' + sig - self.privmsg(nick, 'auth', message) + def send_auth(self, nick, cr): + self.privmsg(nick, 'auth', str(cr)) def send_tx(self, nick_list, txhex): txb64 = base64.b64encode(txhex.decode('hex')) @@ -234,10 +847,22 @@ def privmsg(self, nick, cmd, message): ', dropping message') return message = encrypt_encode(message, box) + + #Anti-replay measure: append the message channel identifier + #to the signature; this prevents cross-channel replay but NOT + #same-channel replay (in case of snooper after dropped connection + #on this channel). + msg_to_be_signed = message + str(self.hostid) + + sig = btc.ecdsa_sign(msg_to_be_signed, self.nick_priv) + message += ' ' + self.nick_pubkey + ' ' + sig #forward to the implementation class (use single _ for polymrphsm to work) self._privmsg(nick, cmd, message) def on_pubmsg(self, nick, message): + #Even illegal messages mark a nick as "seen" + if self.on_pubmsg_trigger: + self.on_pubmsg_trigger(nick, self) if message[0] != COMMAND_PREFIX: return commands = message[1:].split(COMMAND_PREFIX) @@ -248,6 +873,8 @@ def on_pubmsg(self, nick, message): _chunks = command.split(" ") if self.check_for_orders(nick, _chunks): pass + if self.check_for_commitments(nick, _chunks): + pass elif _chunks[0] == 'cancel': # !cancel [oid] try: @@ -259,12 +886,28 @@ def on_pubmsg(self, nick, message): return elif _chunks[0] == 'orderbook': if self.on_orderbook_requested: - self.on_orderbook_requested(nick) + self.on_orderbook_requested(nick, self) else: # TODO this is for testing/debugging, should be removed, see taker.py if hasattr(self, 'debug_on_pubmsg_cmd'): self.debug_on_pubmsg_cmd(nick, _chunks) + def verify_nick(self, nick, sig, message): + if not btc.ecdsa_verify(message + str(self.hostid), sig[1], sig[0]): + log.debug("nick signature verification failed, ignoring.") + return False + #check that nick matches hash of pubkey + nick_pkh_raw = hashlib.sha256(sig[0]).digest()[:NICK_HASH_LENGTH] + nick_stripped = nick[2:2+NICK_MAX_ENCODED] + #strip right padding + nick_unpadded = ''.join([x for x in nick_stripped if x != 'O']) + if not nick_unpadded == btc.changebase(nick_pkh_raw, 256, 58): + log.debug("Nick hash check failed, expected: " + str( + nick_unpadded) + ", got: " + str( + btc.changebase(nick_pkh_raw, 256, 58))) + return False + return True + def on_privmsg(self, nick, message): """handles the case when a private message is received""" #Aberrant short messages should be handled by subclasses @@ -282,7 +925,26 @@ def on_privmsg(self, nick, message): if cmd_string not in plaintext_commands + encrypted_commands: log.debug('cmd not in cmd_list, line="' + message + '"') return - for command in message[1:].split(COMMAND_PREFIX): + #Verify nick ownership + sig = message[1:].split(' ')[-2:] + #reconstruct original message without cmd + rawmessage = ' '.join(message[1:].split(' ')[1:-2]) + #sanity check that the sig was appended properly + if len(sig) != 2 or len(rawmessage) == 0: + log.debug("Sig not properly appended to privmsg, ignoring") + return + if not self.verify_nick(nick, sig, rawmessage): + #This is an impostor; just ignore + log.debug("Message received from unverified counterparty; ignoring") + return + + #Marks the nick as active on this channel; note *only* if verified. + #Otherwise squatter/attacker can persuade us to send privmsgs to him. + if self.on_privmsg_trigger: + self.on_privmsg_trigger(nick, self) + #strip sig from message for processing, having verified + message = " ".join(message[1:].split(" ")[:-2]) + for command in message.split(COMMAND_PREFIX): _chunks = command.split(" ") #Decrypt if necessary @@ -312,7 +974,6 @@ def on_privmsg(self, nick, message): # orderbook watch commands if self.check_for_orders(nick, _chunks): pass - # taker commands elif _chunks[0] == 'pubkey': maker_pk = _chunks[1] @@ -320,35 +981,41 @@ def on_privmsg(self, nick, message): self.on_pubkey(nick, maker_pk) elif _chunks[0] == 'ioauth': utxo_list = _chunks[1].split(',') - cj_pub = _chunks[2] - change_addr = _chunks[3] - btc_sig = _chunks[4] + auth_pub = _chunks[2] + cj_addr = _chunks[3] + change_addr = _chunks[4] + btc_sig = _chunks[5] if self.on_ioauth: - self.on_ioauth(nick, utxo_list, cj_pub, change_addr, - btc_sig) + self.on_ioauth(nick, utxo_list, auth_pub, cj_addr, + change_addr, btc_sig) elif _chunks[0] == 'sig': sig = _chunks[1] if self.on_sig: self.on_sig(nick, sig) # maker commands + if self.check_for_commitments(nick, _chunks, private=True): + pass if _chunks[0] == 'fill': try: oid = int(_chunks[1]) amount = int(_chunks[2]) taker_pk = _chunks[3] + if len(_chunks) > 4: + commit = _chunks[4] + else: + commit = None except (ValueError, IndexError) as e: self.send_error(nick, str(e)) if self.on_order_fill: - self.on_order_fill(nick, oid, amount, taker_pk) + self.on_order_fill(nick, oid, amount, taker_pk, commit) elif _chunks[0] == 'auth': try: - i_utxo_pubkey = _chunks[1] - btc_sig = _chunks[2] + cr = _chunks[1] except (ValueError, IndexError) as e: self.send_error(nick, str(e)) if self.on_seen_auth: - self.on_seen_auth(nick, i_utxo_pubkey, btc_sig) + self.on_seen_auth(nick, cr) elif _chunks[0] == 'tx': b64tx = _chunks[1] try: diff --git a/joinmarket/support.py b/joinmarket/support.py index 3cb67341..72f14d1b 100644 --- a/joinmarket/support.py +++ b/joinmarket/support.py @@ -18,19 +18,21 @@ # dateformat='[%Y/%m/%d %H:%M:%S] ') logFormatter = logging.Formatter( - "%(asctime)s [%(threadName)-12.12s] [%(levelname)-5.5s] %(message)s") + "%(asctime)s [%(threadName)-12.12s] [%(levelname)-5.5s] %(message)s") log = logging.getLogger('joinmarket') log.setLevel(logging.DEBUG) -ORDER_KEYS = ['counterparty', 'oid', 'ordertype', 'minsize', 'maxsize', - 'txfee', 'cjfee'] +ORDER_KEYS = ['counterparty', 'oid', 'ordertype', 'minsize', 'maxsize', 'txfee', + 'cjfee'] joinmarket_alert = [''] core_alert = [''] debug_silence = [False] + #consoleHandler = logging.StreamHandler(stream=sys.stdout) class JoinMarketStreamHandler(logging.StreamHandler): + def __init__(self, stream): super(JoinMarketStreamHandler, self).__init__(stream) @@ -42,6 +44,7 @@ def emit(self, record): if not debug_silence[0]: super(JoinMarketStreamHandler, self).emit(record) + consoleHandler = JoinMarketStreamHandler(stream=sys.stdout) consoleHandler.setFormatter(logFormatter) log.addHandler(consoleHandler) @@ -51,6 +54,7 @@ def emit(self, record): log.debug('hello joinmarket') + def get_log(): """ provides joinmarket logging instance @@ -66,6 +70,7 @@ def get_log(): Only for sampling purposes """ + def rand_norm_array(mu, sigma, n): # use normalvariate instead of gauss for thread safety return [random.normalvariate(mu, sigma) for _ in range(n)] @@ -79,7 +84,7 @@ def rand_exp_array(lamda, n): def rand_pow_array(power, n): # rather crude in that uses a uniform sample which is a multiple of 1e-4 # for basis of formula, see: http://mathworld.wolfram.com/RandomNumber.html - return [y ** (1.0 / power) + return [y**(1.0 / power) for y in [x * 0.0001 for x in random.sample( xrange(10000), n)]] @@ -100,13 +105,13 @@ def rand_weighted_choice(n, p_arr): r = random.random() return sorted(cum_pr + [r]).index(r) - # End random functions def chunks(d, n): return [d[x:x + n] for x in xrange(0, len(d), n)] + def select_gradual(unspent, value): """ UTXO selection algorithm for gradual dust reduction @@ -184,11 +189,11 @@ def select_greediest(unspent, value): def calc_cj_fee(ordertype, cjfee, cj_amount): - if ordertype == 'absorder': + if ordertype == 'absoffer': real_cjfee = int(cjfee) - elif ordertype == 'relorder': - real_cjfee = int( - (Decimal(cjfee) * Decimal(cj_amount)).quantize(Decimal(1))) + elif ordertype == 'reloffer': + real_cjfee = int((Decimal(cjfee) * Decimal(cj_amount)).quantize(Decimal( + 1))) else: raise RuntimeError('unknown order type: ' + str(ordertype)) return real_cjfee @@ -235,8 +240,8 @@ def cheapest_order_choose(orders, n): def pick_order(orders, n): print("Considered orders:") for i, o in enumerate(orders): - print(" %2d. %20s, CJ fee: %6s, tx fee: %6d" % (i, - o[0]['counterparty'], str(o[0]['cjfee']), o[0]['txfee'])) + print(" %2d. %20s, CJ fee: %6s, tx fee: %6d" % + (i, o[0]['counterparty'], str(o[0]['cjfee']), o[0]['txfee'])) pickedOrderIndex = -1 if i == 0: print("Only one possible pick, picking it.") @@ -261,39 +266,44 @@ def choose_orders(db, cj_amount, n, chooseOrdersBy, ignored_makers=None): 'SELECT * FROM orderbook WHERE minsize <= :cja AND :cja <= maxsize;', {'cja': cj_amount}).fetchall() orders = [dict([(k, o[k]) for k in ORDER_KEYS]) - for o in sqlorders if o['counterparty'] not in ignored_makers] - orders_fees = [(o, calc_cj_fee(o['ordertype'], o['cjfee'], - cj_amount) - o['txfee']) for o in orders] + for o in sqlorders if o['counterparty'] not in ignored_makers] + orders_fees = [( + o, calc_cj_fee(o['ordertype'], o['cjfee'], cj_amount) - o['txfee']) + for o in orders] counterparties = set([o['counterparty'] for o in orders]) if n > len(counterparties): log.debug(('ERROR not enough liquidity in the orderbook n=%d ' - 'suitable-counterparties=%d amount=%d totalorders=%d') - % (n, len(counterparties), cj_amount, len(orders))) + 'suitable-counterparties=%d amount=%d totalorders=%d') % + (n, len(counterparties), cj_amount, len(orders))) # TODO handle not enough liquidity better, maybe an Exception return None, 0 - """ restrict to one order per counterparty, choose the one with the lowest cjfee this is done in advance of the order selection algo, so applies to all of them. however, if orders are picked manually, allow duplicates. """ - feekey = lambda x : x[1] + feekey = lambda x: x[1] if chooseOrdersBy != pick_order: orders_fees = sorted( - dict((v[0]['counterparty'], v) for v in sorted(orders_fees, - key=feekey, reverse=True)).values(), key=feekey) + dict((v[0]['counterparty'], v) + for v in sorted(orders_fees, + key=feekey, + reverse=True)).values(), + key=feekey) else: - orders_fees = sorted(orders_fees, key=feekey) #sort by ascending cjfee + orders_fees = sorted(orders_fees, key=feekey) #sort by ascending cjfee - log.debug('considered orders = \n' + '\n'.join([str(o) for o in orders_fees])) + log.debug('considered orders = \n' + '\n'.join([str(o) for o in orders_fees + ])) total_cj_fee = 0 chosen_orders = [] for i in range(n): chosen_order, chosen_fee = chooseOrdersBy(orders_fees, n) - # remove all orders from that same counterparty - orders_fees = [o for o in orders_fees if o[0]['counterparty'] != - chosen_order['counterparty']] + # remove all orders from that same counterparty + orders_fees = [o + for o in orders_fees + if o[0]['counterparty'] != chosen_order['counterparty']] chosen_orders.append(chosen_order) total_cj_fee += chosen_fee log.debug('chosen orders = \n' + '\n'.join([str(o) for o in chosen_orders])) @@ -312,13 +322,13 @@ def choose_sweep_orders(db, i.e. sweep an entire group of utxos solve for cjamount when mychange = 0 - for an order with many makers, a mixture of absorder and relorder + for an order with many makers, a mixture of absoffer and reloffer mychange = totalin - cjamount - total_txfee - sum(absfee) - sum(relfee*cjamount) => 0 = totalin - mytxfee - sum(absfee) - cjamount*(1 + sum(relfee)) => cjamount = (totalin - mytxfee - sum(absfee)) / (1 + sum(relfee)) """ - total_txfee = txfee*n - + total_txfee = txfee * n + if ignored_makers is None: ignored_makers = [] @@ -328,13 +338,13 @@ def calc_zero_change_cj_amount(ordercombo): sumtxfee_contribution = 0 for order in ordercombo: sumtxfee_contribution += order['txfee'] - if order['ordertype'] == 'absorder': + if order['ordertype'] == 'absoffer': sumabsfee += int(order['cjfee']) - elif order['ordertype'] == 'relorder': + elif order['ordertype'] == 'reloffer': sumrelfee += Decimal(order['cjfee']) else: - raise RuntimeError('unknown order type: {}'.format( - order['ordertype'])) + raise RuntimeError('unknown order type: {}'.format(order[ + 'ordertype'])) my_txfee = max(total_txfee - sumtxfee_contribution, 0) cjamount = (total_input_value - my_txfee - sumabsfee) / (1 + sumrelfee) @@ -342,7 +352,7 @@ def calc_zero_change_cj_amount(ordercombo): return cjamount, int(sumabsfee + sumrelfee * cjamount) log.debug('choosing sweep orders for total_input_value = ' + str( - total_input_value) + ' n=' + str(n)) + total_input_value) + ' n=' + str(n)) sqlorders = db.execute('SELECT * FROM orderbook WHERE minsize <= ?;', (total_input_value,)).fetchall() orderlist = [dict([(k, o[k]) for k in ORDER_KEYS]) @@ -350,18 +360,18 @@ def calc_zero_change_cj_amount(ordercombo): log.debug('orderlist = \n' + '\n'.join([str(o) for o in orderlist])) orders_fees = [(o, calc_cj_fee(o['ordertype'], o['cjfee'], - total_input_value)) for o in orderlist] + total_input_value)) for o in orderlist] - feekey = lambda x : x[1] + feekey = lambda x: x[1] # sort from smallest to biggest cj fee orders_fees = sorted(orders_fees, key=feekey) chosen_orders = [] while len(chosen_orders) < n: - if len(orders_fees) < n - len(chosen_orders): - log.debug('ERROR not enough liquidity in the orderbook') - # TODO handle not enough liquidity better, maybe an Exception - return None, 0, 0 for i in range(n - len(chosen_orders)): + if len(orders_fees) < n - len(chosen_orders): + log.debug('ERROR not enough liquidity in the orderbook') + # TODO handle not enough liquidity better, maybe an Exception + return None, 0, 0 chosen_order, chosen_fee = chooseOrdersBy(orders_fees, n) log.debug('chosen = ' + str(chosen_order)) # remove all orders from that same counterparty @@ -369,7 +379,7 @@ def calc_zero_change_cj_amount(ordercombo): o for o in orders_fees if o[0]['counterparty'] != chosen_order['counterparty'] - ] + ] chosen_orders.append(chosen_order) # calc cj_amount and check its in range cj_amount, total_fee = calc_zero_change_cj_amount(chosen_orders) diff --git a/joinmarket/taker.py b/joinmarket/taker.py index a97d0cf8..17adccda 100644 --- a/joinmarket/taker.py +++ b/joinmarket/taker.py @@ -8,10 +8,11 @@ import sys import time import threading +import json from decimal import InvalidOperation, Decimal import bitcoin as btc -from joinmarket.configure import jm_single, get_p2pk_vbyte +from joinmarket.configure import jm_single, get_p2pk_vbyte, donation_address from joinmarket.enc_wrapper import init_keypair, as_init_encryption, init_pubkey, \ NaclError from joinmarket.support import get_log, calc_cj_fee @@ -20,7 +21,6 @@ log = get_log() - class CoinJoinTX(object): # soon the taker argument will be removed and just be replaced by wallet # or some other interface @@ -36,7 +36,8 @@ def __init__(self, total_txfee, finishcallback, choose_orders_recover, - auth_addr=None): + commitment_creator + ): """ if my_change is None then there wont be a change address thats used if you want to entirely coinjoin one utxo with no change left over @@ -59,7 +60,7 @@ def __init__(self, self.my_cj_addr = my_cj_addr self.my_change_addr = my_change_addr self.choose_orders_recover = choose_orders_recover - self.auth_addr = auth_addr + self.commitment_creator = commitment_creator self.timeout_lock = threading.Condition() # used to wait() and notify() # used to restrict access to certain variables across threads self.timeout_thread_lock = threading.Condition() @@ -79,8 +80,41 @@ def __init__(self, # create DH keypair on the fly for this Tx object self.kp = init_keypair() self.crypto_boxes = {} + self.get_commitment(input_utxos, self.cj_amount) self.msgchan.fill_orders(self.active_orders, self.cj_amount, - self.kp.hex_pk()) + self.kp.hex_pk(), self.commitment) + + def get_commitment(self, utxos, amount): + """Create commitment to fulfil anti-DOS requirement of makers, + storing the corresponding reveal/proof data for next step. + """ + while True: + self.commitment, self.reveal_commitment = self.commitment_creator( + self.wallet, utxos, amount) + if (self.commitment) or (jm_single().wait_for_commitments == 0): + break + log.debug("Failed to source commitments, waiting 3 minutes") + time.sleep(3 * 60) + if not self.commitment: + log.debug("Cannot construct transaction, failed to generate " + "commitment, shutting down. Please read commitments_debug.txt " + "for some information on why this is, and what can be " + "done to remedy it.") + #TODO: would like to raw_input here to show the user, but + #interactivity is undesirable here. + #Test only: + if jm_single().config.get( + "BLOCKCHAIN", "blockchain_source") == 'regtest': + raise btc.PoDLEError("For testing raising podle exception") + #The timeout/recovery code is designed to handle non-responsive + #counterparties, but this condition means that the current bot + #is not able to create transactions following its *own* rules, + #so shutting down is appropriate no matter what style + #of bot this is. + #These two settings shut down the timeout thread and avoid recovery. + self.all_responded = True + self.end_timeout_thread = True + self.msgchan.shutdown() def start_encryption(self, nick, maker_pk): if nick not in self.active_orders.keys(): @@ -93,27 +127,24 @@ def start_encryption(self, nick, maker_pk): log.debug("Unable to setup crypto box with " + nick + ": " + repr(e)) self.msgchan.send_error(nick, "invalid nacl pubkey: " + maker_pk) return - # send authorisation request - if self.auth_addr: - my_btc_addr = self.auth_addr - else: - my_btc_addr = self.input_utxos.itervalues().next()['address'] - my_btc_priv = self.wallet.get_key_from_addr(my_btc_addr) - my_btc_pub = btc.privtopub(my_btc_priv) - my_btc_sig = btc.ecdsa_sign(self.kp.hex_pk(), my_btc_priv) - self.msgchan.send_auth(nick, my_btc_pub, my_btc_sig) - def auth_counterparty(self, nick, btc_sig, cj_pub): + self.msgchan.send_auth(nick, self.reveal_commitment) + + def auth_counterparty(self, nick, btc_sig, auth_pub): """Validate the counterpartys claim to own the btc address/pubkey that will be used for coinjoining - with an ecdsa verification.""" - # crypto_boxes[nick][0] = maker_pubkey - if not btc.ecdsa_verify(self.crypto_boxes[nick][0], btc_sig, cj_pub): + with an ecdsa verification. + Note that this is only a first-step + authorisation; it checks the btc signature, but + the authorising pubkey is checked to be part of the + transactoin in recv_txio. + """ + if not btc.ecdsa_verify(self.crypto_boxes[nick][0], btc_sig, auth_pub): log.debug('signature didnt match pubkey and message') return False return True - def recv_txio(self, nick, utxo_list, cj_pub, change_addr): + def recv_txio(self, nick, utxo_list, auth_pub, cj_addr, change_addr): if nick not in self.nonrespondants: log.debug(('recv_txio => nick={} not in ' 'nonrespondants {}').format(nick, self.nonrespondants)) @@ -126,6 +157,16 @@ def recv_txio(self, nick, utxo_list, cj_pub, change_addr): # when internal reviewing of makers is created, add it here to # immediately quit; currently, the timeout thread suffices. return + #Complete maker authorization: + #Extract the address fields from the utxos + #Construct the Bitcoin address for the auth_pub field + #Ensure that at least one address from utxos corresponds. + input_addresses = [d['address'] for d in utxo_data] + auth_address = btc.pubkey_to_address(auth_pub, get_p2pk_vbyte()) + if not auth_address in input_addresses: + log.debug("ERROR maker's authorising pubkey is not included " + "in the transaction: " + str(auth_address)) + return total_input = sum([d['value'] for d in utxo_data]) real_cjfee = calc_cj_fee(self.active_orders[nick]['ordertype'], @@ -148,7 +189,6 @@ def recv_txio(self, nick, utxo_list, cj_pub, change_addr): 'cjamount={:d} txfee={:d} realcjfee={:d}').format log.debug(fmt(nick, total_input, self.cj_amount, self.active_orders[nick]['txfee'], real_cjfee)) - cj_addr = btc.pubtoaddr(cj_pub, get_p2pk_vbyte()) self.outputs.append({'address': cj_addr, 'value': self.cj_amount}) self.cjfee_total += real_cjfee self.maker_txfee_contributions += self.active_orders[nick]['txfee'] @@ -302,13 +342,14 @@ def coinjoin_address(self): if self.my_cj_addr: return self.my_cj_addr else: - return donation_address(self) + addr, self.sign_k = donation_address() + return addr def sign_tx(self, tx, i, priv): if self.my_cj_addr: return btc.sign(tx, i, priv) else: - return sign_donation_tx(tx, i, priv) + return btc.sign(tx, i, priv, usenonce=btc.safe_from_hex(self.sign_k)) def self_sign(self): # now sign it ourselves @@ -387,9 +428,10 @@ def recover_from_nonrespondants(self): '{}').format( pprint.pformat(self.active_orders), pprint.pformat(self.nonrespondants))) - + #Re-source commitment; previous attempt will have been blacklisted + self.get_commitment(self.input_utxos, self.cj_amount) self.msgchan.fill_orders(new_orders, self.cj_amount, - self.kp.hex_pk()) + self.kp.hex_pk(), self.commitment) else: log.debug('nonresponse to !tx') # nonresponding to !tx, have to restart tx from the beginning @@ -465,7 +507,7 @@ def __init__(self, msgchan): self.msgchan.register_channel_callbacks( self.on_welcome, self.on_set_topic, None, self.on_disconnect, self.on_nick_leave, None) - + self.dblock = threading.Lock() con = sqlite3.connect(":memory:", check_same_thread=False) con.row_factory = sqlite3.Row self.db = con.cursor() @@ -476,6 +518,7 @@ def __init__(self, msgchan): def on_order_seen(self, counterparty, oid, ordertype, minsize, maxsize, txfee, cjfee): try: + self.dblock.acquire(True) if int(oid) < 0 or int(oid) > sys.maxint: log.debug( "Got invalid order ID: " + oid + " from " + counterparty) @@ -509,12 +552,12 @@ def on_order_seen(self, counterparty, oid, ordertype, minsize, maxsize, "from {}").format log.debug(fmt(minsize, maxsize, counterparty)) return - if ordertype == 'absorder' and not isinstance(cjfee, int): + if ordertype == 'absoffer' and not isinstance(cjfee, int): try: cjfee = int(cjfee) except ValueError: log.debug("Got non integer coinjoin fee: " + str(cjfee) + - " for an absorder from " + counterparty) + " for an absoffer from " + counterparty) return self.db.execute( 'INSERT INTO orderbook VALUES(?, ?, ?, ?, ?, ?, ?);', @@ -523,21 +566,27 @@ def on_order_seen(self, counterparty, oid, ordertype, minsize, maxsize, cjfee)))) # any parseable Decimal is a valid cjfee except InvalidOperation: log.debug("Got invalid cjfee: " + cjfee + " from " + counterparty) - except: + except Exception as e: log.debug("Error parsing order " + oid + " from " + counterparty) + log.debug("Exception was: " + repr(e)) + finally: + self.dblock.release() def on_order_cancel(self, counterparty, oid): - self.db.execute(("DELETE FROM orderbook WHERE " + with self.dblock: + self.db.execute(("DELETE FROM orderbook WHERE " "counterparty=? AND oid=?;"), (counterparty, oid)) def on_welcome(self): self.msgchan.request_orderbook() def on_nick_leave(self, nick): - self.db.execute('DELETE FROM orderbook WHERE counterparty=?;', (nick,)) + with self.dblock: + self.db.execute('DELETE FROM orderbook WHERE counterparty=?;', (nick,)) def on_disconnect(self): - self.db.execute('DELETE FROM orderbook;') + with self.dblock: + self.db.execute('DELETE FROM orderbook;') # assume this only has one open cj tx at a time @@ -546,14 +595,14 @@ def __init__(self, msgchan): OrderbookWatch.__init__(self, msgchan) msgchan.register_taker_callbacks(self.on_error, self.on_pubkey, self.on_ioauth, self.on_sig) - msgchan.cjpeer = self + msgchan.set_cjpeer(self) self.cjtx = None self.maker_pks = {} # TODO have a list of maker's nick we're coinjoining with, so # that some other guy doesnt send you confusing stuff def get_crypto_box_from_nick(self, nick): - if nick in self.cjtx.crypto_boxes: + if nick in self.cjtx.crypto_boxes and self.cjtx.crypto_boxes[nick] != None: return self.cjtx.crypto_boxes[nick][ 1] # libsodium encryption object else: @@ -570,14 +619,19 @@ def start_cj(self, my_change_addr, total_txfee, finishcallback=None, - choose_orders_recover=None, - auth_addr=None): + choose_orders_recover=None + ): self.cjtx = None + #needed during commitment preparation, self.cjtx.cj_amount + #will be the amount after CoinJoinTx.__init__() completes. + #(and same for self.cjtx.wallet) + self.proposed_cj_amount = cj_amount + self.proposed_wallet = wallet self.cjtx = CoinJoinTX( self.msgchan, wallet, self.db, cj_amount, orders, input_utxos, my_cj_addr, my_change_addr, total_txfee, finishcallback, - choose_orders_recover, auth_addr) + choose_orders_recover, self.make_commitment) def on_error(self): pass # TODO implement @@ -589,67 +643,153 @@ def on_pubkey(self, nick, maker_pubkey): time.sleep(0.5) self.cjtx.start_encryption(nick, maker_pubkey) - def on_ioauth(self, nick, utxo_list, cj_pub, change_addr, btc_sig): - if not self.cjtx.auth_counterparty(nick, btc_sig, cj_pub): + def on_ioauth(self, nick, utxo_list, auth_pub, cj_addr, change_addr, btc_sig): + if not self.cjtx.auth_counterparty(nick, btc_sig, auth_pub): fmt = ('Authenticated encryption with counterparty: {}' ' not established. TODO: send rejection message').format log.debug(fmt(nick)) return with self.cjtx.timeout_thread_lock: - self.cjtx.recv_txio(nick, utxo_list, cj_pub, change_addr) + self.cjtx.recv_txio(nick, utxo_list, auth_pub, cj_addr, change_addr) def on_sig(self, nick, sig): with self.cjtx.timeout_thread_lock: self.cjtx.add_signature(nick, sig) + def make_commitment(self, wallet, input_utxos, cjamount): + """The Taker default commitment function, which uses PoDLE. + Alternative commitment types should use a different commit type byte. + This will allow future upgrades to provide different style commitments + by subclassing Taker and changing the commit_type_byte; existing makers + will simply not accept this new type of commitment. + In case of success, return the commitment and its opening. + In case of failure returns (None, None) and constructs a detailed + log for the user to read and discern the reason. + """ + + def filter_by_coin_age_amt(utxos, age, amt): + results = jm_single().bc_interface.query_utxo_set(utxos, + includeconf=True) + newresults = [] + too_old = [] + too_small = [] + for i, r in enumerate(results): + #results return "None" if txo is spent; drop this + if not r: + continue + valid_age = r['confirms'] >= age + valid_amt = r['value'] >= amt + if not valid_age: + too_old.append(utxos[i]) + if not valid_amt: + too_small.append(utxos[i]) + if valid_age and valid_amt: + newresults.append(utxos[i]) + + return newresults, too_old, too_small + + def priv_utxo_pairs_from_utxos(utxos, age, amt): + #returns pairs list of (priv, utxo) for each valid utxo; + #also returns lists "too_old" and "too_small" for any + #utxos that did not satisfy the criteria for debugging. + priv_utxo_pairs = [] + new_utxos, too_old, too_small = filter_by_coin_age_amt( + utxos.keys(), age, amt) + new_utxos_dict = {k: v for k, v in utxos.items() if k in new_utxos} + for k, v in new_utxos_dict.iteritems(): + addr = v['address'] + priv = wallet.get_key_from_addr(addr) + if priv: #can be null from create-unsigned + priv_utxo_pairs.append((priv, k)) + return priv_utxo_pairs, too_old, too_small + + commit_type_byte = "P" + podle_data = None + tries = jm_single().config.getint("POLICY", "taker_utxo_retries") + age = jm_single().config.getint("POLICY", "taker_utxo_age") + #Minor rounding errors don't matter here + amt = int(cjamount * jm_single().config.getint( + "POLICY", "taker_utxo_amtpercent") / 100.0) + priv_utxo_pairs, to, ts = priv_utxo_pairs_from_utxos(input_utxos, age, amt) + #Note that we ignore the "too old" and "too small" lists in the first + #pass through, because the same utxos appear in the whole-wallet check. + + #For podle data format see: btc.podle.PoDLE.reveal() + #In first round try, don't use external commitments + podle_data = btc.generate_podle(priv_utxo_pairs, tries) + if not podle_data: + #We defer to a second round to try *all* utxos in wallet; + #this is because it's much cleaner to use the utxos involved + #in the transaction, about to be consumed, rather than use + #random utxos that will persist after. At this step we also + #allow use of external utxos in the json file. + if wallet.unspent: + priv_utxo_pairs, to, ts = priv_utxo_pairs_from_utxos( + wallet.unspent, age, amt) + #Pre-filter the set of external commitments that work for this + #transaction according to its size and age. + dummy, extdict = btc.get_podle_commitments() + if len(extdict.keys()) > 0: + ext_valid, ext_to, ext_ts = filter_by_coin_age_amt(extdict.keys(), + age, amt) + else: + ext_valid = None + podle_data = btc.generate_podle(priv_utxo_pairs, tries, ext_valid) + if podle_data: + log.debug("Generated PoDLE: " + pprint.pformat(podle_data)) + revelation = btc.PoDLE(u=podle_data['utxo'],P=podle_data['P'], + P2=podle_data['P2'],s=podle_data['sig'], + e=podle_data['e']).serialize_revelation() + return (commit_type_byte + podle_data["commit"], revelation) + else: + #we know that priv_utxo_pairs all passed age and size tests, so + #they must have failed the retries test. Summarize this info + #and publish to commitments_debug.txt + with open("commitments_debug.txt", "wb") as f: + f.write("THIS IS A TEMPORARY FILE FOR DEBUGGING; " + "IT CAN BE SAFELY DELETED ANY TIME.\n") + f.write("***\n") + f.write("1: Utxos that passed age and size limits, but have " + "been used too many times (see taker_utxo_retries " + "in the config):\n") + if len(priv_utxo_pairs) == 0: + f.write("None\n") + else: + for p, u in priv_utxo_pairs: + f.write(str(u) + "\n") + f.write("2: Utxos that have less than " + jm_single().config.get( + "POLICY", "taker_utxo_age") + " confirmations:\n") + if len(to) == 0: + f.write("None\n") + else: + for t in to: + f.write(str(t) + "\n") + f.write("3: Utxos that were not at least " + \ + jm_single().config.get( + "POLICY", "taker_utxo_amtpercent") + "% of the " + "size of the coinjoin amount " + str( + self.proposed_cj_amount) + "\n") + if len(ts) == 0: + f.write("None\n") + else: + for t in ts: + f.write(str(t) + "\n") + f.write('***\n') + f.write("Utxos that appeared in item 1 cannot be used again.\n") + f.write("Utxos only in item 2 can be used by waiting for more " + "confirmations, (set by the value of taker_utxo_age).\n") + f.write("Utxos only in item 3 are not big enough for this " + "coinjoin transaction, set by the value " + "of taker_utxo_amtpercent.\n") + f.write("If you cannot source a utxo from your wallet according " + "to these rules, use the tool add-utxo.py to source a " + "utxo external to your joinmarket wallet. Read the help " + "with 'python add-utxo.py --help'\n\n") + f.write("You can also reset the rules in the joinmarket.cfg " + "file, but this is generally inadvisable.\n") + f.write("***\nFor reference, here are the utxos in your wallet:\n") + f.write("\n" + str(self.proposed_wallet.unspent)) + + return (None, None) + -# this stuff copied and slightly modified from pybitcointools -def donation_address(cjtx): - from bitcoin.main import multiply, G, deterministic_generate_k, add_pubkeys - reusable_donation_pubkey = ('02be838257fbfddabaea03afbb9f16e852' - '9dfe2de921260a5c46036d97b5eacf2a') - - donation_utxo_data = cjtx.input_utxos.iteritems().next() - global donation_utxo - donation_utxo = donation_utxo_data[0] - privkey = cjtx.wallet.get_key_from_addr(donation_utxo_data[1]['address']) - # tx without our inputs and outputs - tx = btc.mktx(cjtx.utxo_tx, cjtx.outputs) - msghash = btc.bin_txhash(tx, btc.SIGHASH_ALL) - # generate unpredictable k - global sign_k - sign_k = deterministic_generate_k(msghash, privkey) - c = btc.sha256(multiply(reusable_donation_pubkey, sign_k)) - sender_pubkey = add_pubkeys( - reusable_donation_pubkey, multiply( - G, c)) - sender_address = btc.pubtoaddr(sender_pubkey, get_p2pk_vbyte()) - log.debug('sending coins to ' + sender_address) - return sender_address - - -def sign_donation_tx(tx, i, priv): - from bitcoin.main import fast_multiply, decode_privkey, G, inv, N - from bitcoin.transaction import der_encode_sig - k = sign_k - hashcode = btc.SIGHASH_ALL - i = int(i) - if len(priv) <= 33: - priv = btc.safe_hexlify(priv) - pub = btc.privkey_to_pubkey(priv) - address = btc.pubkey_to_address(pub) - signing_tx = btc.signature_form( - tx, i, btc.mk_pubkey_script(address), hashcode) - - msghash = btc.bin_txhash(signing_tx, hashcode) - z = btc.hash_to_int(msghash) - # k = deterministic_generate_k(msghash, priv) - r, y = fast_multiply(G, k) - s = inv(k, N) * (z + r * decode_privkey(priv)) % N - rawsig = 27 + (y % 2), r, s - - sig = der_encode_sig(*rawsig) + btc.encode(hashcode, 16, 2) - # sig = ecdsa_tx_sign(signing_tx, priv, hashcode) - txobj = btc.deserialize(tx) - txobj["ins"][i]["script"] = btc.serialize_script([sig, pub]) - return btc.serialize(txobj) diff --git a/joinmarket/wallet.py b/joinmarket/wallet.py index fd37d78f..7b05a107 100644 --- a/joinmarket/wallet.py +++ b/joinmarket/wallet.py @@ -200,13 +200,10 @@ def read_wallet_file_data(self, filename, pwd=None): epk_m['encrypted_privkey'].decode( 'hex')).encode('hex') #Imported keys are stored as 32 byte strings only, so the #second version below is sufficient, really. - if not btc.secp_present: - privkey = btc.encode_privkey(privkey, 'hex_compressed') - else: - if len(privkey) != 64: - raise Exception( - "Unexpected privkey format; already compressed?:" + privkey) - privkey += "01" + if len(privkey) != 64: + raise Exception( + "Unexpected privkey format; already compressed?:" + privkey) + privkey += "01" if epk_m['mixdepth'] not in self.imported_privkeys: self.imported_privkeys[epk_m['mixdepth']] = [] self.addr_cache[btc.privtoaddr( diff --git a/joinmarket/yieldgenerator.py b/joinmarket/yieldgenerator.py new file mode 100644 index 00000000..2a915366 --- /dev/null +++ b/joinmarket/yieldgenerator.py @@ -0,0 +1,161 @@ +#! /usr/bin/env python +from __future__ import absolute_import, print_function + +import datetime +import os +import time +import abc +from optparse import OptionParser + +from joinmarket import Maker, IRCMessageChannel, MessageChannelCollection +from joinmarket import BlockrInterface +from joinmarket import jm_single, get_network, load_program_config +from joinmarket import get_log, calc_cj_fee, debug_dump_object +from joinmarket import Wallet +from joinmarket import get_irc_mchannels + +log = get_log() + +# is a maker for the purposes of generating a yield from held +# bitcoins +class YieldGenerator(Maker): + __metaclass__ = abc.ABCMeta + statement_file = os.path.join('logs', 'yigen-statement.csv') + + def __init__(self, msgchan, wallet): + Maker.__init__(self, msgchan, wallet) + self.msgchan.register_channel_callbacks(self.on_welcome, + self.on_set_topic, None, None, + self.on_nick_leave, None) + self.tx_unconfirm_timestamp = {} + + def log_statement(self, data): + if get_network() == 'testnet': + return + + data = [str(d) for d in data] + self.income_statement = open(self.statement_file, 'a') + self.income_statement.write(','.join(data) + '\n') + self.income_statement.close() + + def on_welcome(self): + Maker.on_welcome(self) + if not os.path.isfile(self.statement_file): + self.log_statement( + ['timestamp', 'cj amount/satoshi', 'my input count', + 'my input value/satoshi', 'cjfee/satoshi', 'earned/satoshi', + 'confirm time/min', 'notes']) + + timestamp = datetime.datetime.now().strftime("%Y/%m/%d %H:%M:%S") + self.log_statement([timestamp, '', '', '', '', '', '', 'Connected']) + + @abc.abstractmethod + def create_my_orders(self): + """Must generate a set of orders to be displayed + according to the contents of the wallet + some algo. + (Note: should be called "create_my_offers") + """ + + @abc.abstractmethod + def oid_to_order(self, cjorder, oid, amount): + """Must convert an order with an offer/order id + into a set of utxos to fill the order. + Also provides the output addresses for the Taker. + """ + + @abc.abstractmethod + def on_tx_unconfirmed(self, cjorder, txid, removed_utxos): + """Performs action on receipt of transaction into the + mempool in the blockchain instance (e.g. announcing orders) + """ + + @abc.abstractmethod + def on_tx_confirmed(self, cjorder, confirmations, txid): + """Performs actions on receipt of 1st confirmation of + a transaction into a block (e.g. announce orders) + """ + + +def ygmain(ygclass, txfee=1000, cjfee_a=200, cjfee_r=0.002, ordertype='reloffer', + nickserv_password='', minsize=100000, mix_levels=5): + import sys + + parser = OptionParser(usage='usage: %prog [options] [wallet file]') + parser.add_option('-o', '--ordertype', action='store', type='string', + dest='ordertype', default=ordertype, + help='type of order; can be either reloffer or absoffer') + parser.add_option('-t', '--txfee', action='store', type='int', + dest='txfee', default=txfee, + help='minimum miner fee in satoshis') + parser.add_option('-c', '--cjfee', action='store', type='string', + dest='cjfee', default='', + help='requested coinjoin fee in satoshis or proportion') + parser.add_option('-p', '--password', action='store', type='string', + dest='password', default=nickserv_password, + help='irc nickserv password') + parser.add_option('-s', '--minsize', action='store', type='int', + dest='minsize', default=minsize, + help='minimum coinjoin size in satoshis') + parser.add_option('-m', '--mixlevels', action='store', type='int', + dest='mixlevels', default=mix_levels, + help='number of mixdepths to use') + (options, args) = parser.parse_args() + if len(args) < 1: + parser.error('Needs a wallet') + sys.exit(0) + seed = args[0] + ordertype = options.ordertype + txfee = options.txfee + if ordertype == 'reloffer': + if options.cjfee != '': + cjfee_r = options.cjfee + # minimum size is such that you always net profit at least 20% + #of the miner fee + minsize = max(int(1.2 * txfee / float(cjfee_r)), options.minsize) + elif ordertype == 'absoffer': + if options.cjfee != '': + cjfee_a = int(options.cjfee) + minsize = options.minsize + else: + parser.error('You specified an incorrect order type which ' +\ + 'can be either reloffer or absoffer') + sys.exit(0) + nickserv_password = options.password + mix_levels = options.mixlevels + + load_program_config() + if isinstance(jm_single().bc_interface, BlockrInterface): + c = ('\nYou are running a yield generator by polling the blockr.io ' + 'website. This is quite bad for privacy. That site is owned by ' + 'coinbase.com Also your bot will run faster and more efficently, ' + 'you can be immediately notified of new bitcoin network ' + 'information so your money will be working for you as hard as ' + 'possibleLearn how to setup JoinMarket with Bitcoin Core: ' + 'https://github.com/chris-belcher/joinmarket/wiki/Running' + '-JoinMarket-with-Bitcoin-Core-full-node') + print(c) + ret = raw_input('\nContinue? (y/n):') + if ret[0] != 'y': + return + + wallet = Wallet(seed, max_mix_depth=mix_levels) + jm_single().bc_interface.sync_wallet(wallet) + + log.debug('starting yield generator') + mcs = [IRCMessageChannel(c, realname='btcint=' + jm_single().config.get( + "BLOCKCHAIN", "blockchain_source"), + password=nickserv_password) for c in get_irc_mchannels()] + mcc = MessageChannelCollection(mcs) + maker = ygclass(mcc, wallet, [options.txfee, cjfee_a, cjfee_r, + options.ordertype, options.minsize, mix_levels]) + try: + log.debug('connecting to message channels') + mcc.run() + except: + log.debug('CRASHING, DUMPING EVERYTHING') + debug_dump_object(wallet, ['addr_cache', 'keys', 'seed']) + debug_dump_object(maker) + debug_dump_object(mcc) + import traceback + log.debug(traceback.format_exc()) + diff --git a/libnacl/__init__.py b/libnacl/__init__.py deleted file mode 100644 index e0a0a621..00000000 --- a/libnacl/__init__.py +++ /dev/null @@ -1,553 +0,0 @@ -# -*- coding: utf-8 -*- -''' -Wrap libsodium routines -''' -# pylint: disable=C0103 -# Import libnacl libs -from libnacl.version import __version__ -# Import python libs -import ctypes -import sys - -__SONAMES = (13, 10, 5, 4) - - -def _get_nacl(): - ''' - Locate the nacl c libs to use - ''' - # Import libsodium - if sys.platform.startswith('win'): - try: - return ctypes.cdll.LoadLibrary('libsodium') - except OSError: - pass - for soname_ver in __SONAMES: - try: - return ctypes.cdll.LoadLibrary('libsodium-{0}'.format( - soname_ver)) - except OSError: - pass - try: - return ctypes.cdll.LoadLibrary('tweetnacl') - except OSError: - msg = ('Could not locate nacl lib, searched for libsodium, ' - 'tweetnacl') - raise OSError(msg) - elif sys.platform.startswith('darwin'): - try: - return ctypes.cdll.LoadLibrary('libsodium.dylib') - except OSError: - pass - try: - return ctypes.cdll.LoadLibrary('tweetnacl.dylib') - except OSError: - msg = ('Could not locate nacl lib, searched for libsodium, ' - 'tweetnacl') - raise OSError(msg) - else: - try: - return ctypes.cdll.LoadLibrary('libsodium.so') - except OSError: - pass - try: - return ctypes.cdll.LoadLibrary('/usr/local/lib/libsodium.so') - except OSError: - pass - - for soname_ver in __SONAMES: - try: - return ctypes.cdll.LoadLibrary('libsodium.so.{0}'.format( - soname_ver)) - except OSError: - pass - try: - return ctypes.cdll.LoadLibrary('tweetnacl.so') - except OSError: - msg = 'Could not locate nacl lib, searched for libsodium.so, ' - for soname_ver in __SONAMES: - msg += 'libsodium.so.{0}, '.format(soname_ver) - msg += ' and tweetnacl.so' - raise OSError(msg) - - -nacl = _get_nacl() - -# Define constants -crypto_box_SECRETKEYBYTES = nacl.crypto_box_secretkeybytes() -crypto_box_PUBLICKEYBYTES = nacl.crypto_box_publickeybytes() -crypto_box_NONCEBYTES = nacl.crypto_box_noncebytes() -crypto_box_ZEROBYTES = nacl.crypto_box_zerobytes() -crypto_box_BOXZEROBYTES = nacl.crypto_box_boxzerobytes() -crypto_box_BEFORENMBYTES = nacl.crypto_box_beforenmbytes() -crypto_scalarmult_BYTES = nacl.crypto_scalarmult_bytes() -crypto_scalarmult_SCALARBYTES = nacl.crypto_scalarmult_scalarbytes() -crypto_sign_BYTES = nacl.crypto_sign_bytes() -crypto_sign_SEEDBYTES = nacl.crypto_sign_secretkeybytes() // 2 -crypto_sign_PUBLICKEYBYTES = nacl.crypto_sign_publickeybytes() -crypto_sign_SECRETKEYBYTES = nacl.crypto_sign_secretkeybytes() -crypto_box_MACBYTES = crypto_box_ZEROBYTES - crypto_box_BOXZEROBYTES -crypto_secretbox_KEYBYTES = nacl.crypto_secretbox_keybytes() -crypto_secretbox_NONCEBYTES = nacl.crypto_secretbox_noncebytes() -crypto_secretbox_ZEROBYTES = nacl.crypto_secretbox_zerobytes() -crypto_secretbox_BOXZEROBYTES = nacl.crypto_secretbox_boxzerobytes() -crypto_secretbox_MACBYTES = crypto_secretbox_ZEROBYTES - crypto_secretbox_BOXZEROBYTES -crypto_stream_KEYBYTES = nacl.crypto_stream_keybytes() -crypto_stream_NONCEBYTES = nacl.crypto_stream_noncebytes() -crypto_auth_BYTES = nacl.crypto_auth_bytes() -crypto_auth_KEYBYTES = nacl.crypto_auth_keybytes() -crypto_onetimeauth_BYTES = nacl.crypto_onetimeauth_bytes() -crypto_onetimeauth_KEYBYTES = nacl.crypto_onetimeauth_keybytes() -crypto_generichash_BYTES = nacl.crypto_generichash_bytes() -crypto_generichash_BYTES_MIN = nacl.crypto_generichash_bytes_min() -crypto_generichash_BYTES_MAX = nacl.crypto_generichash_bytes_max() -crypto_generichash_KEYBYTES = nacl.crypto_generichash_keybytes() -crypto_generichash_KEYBYTES_MIN = nacl.crypto_generichash_keybytes_min() -crypto_generichash_KEYBYTES_MAX = nacl.crypto_generichash_keybytes_max() -crypto_scalarmult_curve25519_BYTES = nacl.crypto_scalarmult_curve25519_bytes() -crypto_hash_BYTES = nacl.crypto_hash_sha512_bytes() -crypto_hash_sha256_BYTES = nacl.crypto_hash_sha256_bytes() -crypto_hash_sha512_BYTES = nacl.crypto_hash_sha512_bytes() - -# pylint: enable=C0103 - - -# Define exceptions -class CryptError(Exception): - ''' - Base Exception for cryptographic errors - ''' - -# Pubkey defs - - -def crypto_box_keypair(): - ''' - Generate and return a new keypair - - pk, sk = nacl.crypto_box_keypair() - ''' - pk = ctypes.create_string_buffer(crypto_box_PUBLICKEYBYTES) - sk = ctypes.create_string_buffer(crypto_box_SECRETKEYBYTES) - nacl.crypto_box_keypair(pk, sk) - return pk.raw, sk.raw - - -def crypto_box(msg, nonce, pk, sk): - ''' - Using a public key and a secret key encrypt the given message. A nonce - must also be passed in, never reuse the nonce - - enc_msg = nacl.crypto_box('secret message', , , ) - ''' - if len(pk) != crypto_box_PUBLICKEYBYTES: - raise ValueError('Invalid public key') - if len(sk) != crypto_box_SECRETKEYBYTES: - raise ValueError('Invalid secret key') - if len(nonce) != crypto_box_NONCEBYTES: - raise ValueError('Invalid nonce') - pad = b'\x00' * crypto_box_ZEROBYTES + msg - c = ctypes.create_string_buffer(len(pad)) - ret = nacl.crypto_box(c, pad, ctypes.c_ulonglong(len(pad)), nonce, pk, sk) - if ret: - raise CryptError('Unable to encrypt message') - return c.raw[crypto_box_BOXZEROBYTES:] - - -def crypto_box_open(ctxt, nonce, pk, sk): - ''' - Decrypts a message given the receivers private key, and senders public key - ''' - if len(pk) != crypto_box_PUBLICKEYBYTES: - raise ValueError('Invalid public key') - if len(sk) != crypto_box_SECRETKEYBYTES: - raise ValueError('Invalid secret key') - if len(nonce) != crypto_box_NONCEBYTES: - raise ValueError('Invalid nonce') - pad = b'\x00' * crypto_box_BOXZEROBYTES + ctxt - msg = ctypes.create_string_buffer(len(pad)) - ret = nacl.crypto_box_open(msg, pad, ctypes.c_ulonglong(len(pad)), nonce, - pk, sk) - if ret: - raise CryptError('Unable to decrypt ciphertext') - return msg.raw[crypto_box_ZEROBYTES:] - - -def crypto_box_beforenm(pk, sk): - ''' - Partially performs the computation required for both encryption and decryption of data - ''' - if len(pk) != crypto_box_PUBLICKEYBYTES: - raise ValueError('Invalid public key') - if len(sk) != crypto_box_SECRETKEYBYTES: - raise ValueError('Invalid secret key') - k = ctypes.create_string_buffer(crypto_box_BEFORENMBYTES) - ret = nacl.crypto_box_beforenm(k, pk, sk) - if ret: - raise CryptError('Unable to compute shared key') - return k.raw - - -def crypto_box_afternm(msg, nonce, k): - ''' - Encrypts a given a message, using partial computed data - ''' - if len(k) != crypto_box_BEFORENMBYTES: - raise ValueError('Invalid shared key') - if len(nonce) != crypto_box_NONCEBYTES: - raise ValueError('Invalid nonce') - pad = b'\x00' * crypto_box_ZEROBYTES + msg - ctxt = ctypes.create_string_buffer(len(pad)) - ret = nacl.crypto_box_afternm(ctxt, pad, ctypes.c_ulonglong(len(pad)), - nonce, k) - if ret: - raise ValueError('Unable to encrypt messsage') - return ctxt.raw[crypto_box_BOXZEROBYTES:] - - -def crypto_box_open_afternm(ctxt, nonce, k): - ''' - Decrypts a ciphertext ctxt given k - ''' - if len(k) != crypto_box_BEFORENMBYTES: - raise ValueError('Invalid shared key') - if len(nonce) != crypto_box_NONCEBYTES: - raise ValueError('Invalid nonce') - pad = b'\x00' * crypto_box_BOXZEROBYTES + ctxt - msg = ctypes.create_string_buffer(len(pad)) - ret = nacl.crypto_box_open_afternm(msg, pad, ctypes.c_ulonglong(len(pad)), - nonce, k) - if ret: - raise ValueError('unable to decrypt message') - return msg.raw[crypto_box_ZEROBYTES:] - -# Signing functions - - -def crypto_sign_keypair(): - ''' - Generates a signing/verification key pair - ''' - vk = ctypes.create_string_buffer(crypto_sign_PUBLICKEYBYTES) - sk = ctypes.create_string_buffer(crypto_sign_SECRETKEYBYTES) - ret = nacl.crypto_sign_keypair(vk, sk) - if ret: - raise ValueError('Failed to generate keypair') - return vk.raw, sk.raw - - -def crypto_sign(msg, sk): - ''' - Sign the given message witht he given signing key - ''' - sig = ctypes.create_string_buffer(len(msg) + crypto_sign_BYTES) - slen = ctypes.pointer(ctypes.c_ulonglong()) - ret = nacl.crypto_sign(sig, slen, msg, ctypes.c_ulonglong(len(msg)), sk) - if ret: - raise ValueError('Failed to sign message') - return sig.raw - - -def crypto_sign_seed_keypair(seed): - ''' - Computes and returns the secret adn verify keys from the given seed - ''' - if len(seed) != crypto_sign_SEEDBYTES: - raise ValueError('Invalid Seed') - sk = ctypes.create_string_buffer(crypto_sign_SECRETKEYBYTES) - vk = ctypes.create_string_buffer(crypto_sign_PUBLICKEYBYTES) - - ret = nacl.crypto_sign_seed_keypair(vk, sk, seed) - if ret: - raise CryptError('Failed to generate keypair from seed') - return (vk.raw, sk.raw) - - -def crypto_sign_open(sig, vk): - ''' - Verifies the signed message sig using the signer's verification key - ''' - msg = ctypes.create_string_buffer(len(sig)) - msglen = ctypes.c_ulonglong() - msglenp = ctypes.pointer(msglen) - ret = nacl.crypto_sign_open(msg, msglenp, sig, ctypes.c_ulonglong(len(sig)), - vk) - if ret: - raise ValueError('Failed to validate message') - return msg.raw[:msglen.value] # pylint: disable=invalid-slice-index - -# Authenticated Symmetric Encryption - - -def crypto_secretbox(msg, nonce, key): - ''' - Encrypts and authenticates a message using the given secret key, and nonce - ''' - pad = b'\x00' * crypto_secretbox_ZEROBYTES + msg - ctxt = ctypes.create_string_buffer(len(pad)) - ret = nacl.crypto_secretbox(ctxt, pad, ctypes.c_ulonglong(len(pad)), nonce, - key) - if ret: - raise ValueError('Failed to encrypt message') - return ctxt.raw[crypto_secretbox_BOXZEROBYTES:] - - -def crypto_secretbox_open(ctxt, nonce, key): - ''' - Decrypts a ciphertext ctxt given the receivers private key, and senders - public key - ''' - pad = b'\x00' * crypto_secretbox_BOXZEROBYTES + ctxt - msg = ctypes.create_string_buffer(len(pad)) - ret = nacl.crypto_secretbox_open(msg, pad, ctypes.c_ulonglong(len(pad)), - nonce, key) - if ret: - raise ValueError('Failed to decrypt message') - return msg.raw[crypto_secretbox_ZEROBYTES:] - -# Symmetric Encryption - - -def crypto_stream(slen, nonce, key): - ''' - Generates a stream using the given secret key and nonce - ''' - stream = ctypes.create_string_buffer(slen) - ret = nacl.crypto_stream(stream, ctypes.c_ulonglong(slen), nonce, key) - if ret: - raise ValueError('Failed to init stream') - return stream.raw - - -def crypto_stream_xor(msg, nonce, key): - ''' - Encrypts the given message using the given secret key and nonce - - The crypto_stream_xor function guarantees that the ciphertext is the - plaintext (xor) the output of crypto_stream. Consequently - crypto_stream_xor can also be used to decrypt - ''' - stream = ctypes.create_string_buffer(len(msg)) - ret = nacl.crypto_stream_xor(stream, msg, ctypes.c_ulonglong(len(msg)), - nonce, key) - if ret: - raise ValueError('Failed to init stream') - return stream.raw - -# Authentication - - -def crypto_auth(msg, key): - ''' - Constructs a one time authentication token for the given message msg - using a given secret key - ''' - tok = ctypes.create_string_buffer(crypto_auth_BYTES) - ret = nacl.crypto_auth(tok, msg, ctypes.c_ulonglong(len(msg)), key) - if ret: - raise ValueError('Failed to auth msg') - return tok.raw[:crypto_auth_BYTES] - - -def crypto_auth_verify(msg, key): - ''' - Verifies that the given authentication token is correct for the given - message and key - ''' - tok = ctypes.create_string_buffer(crypto_auth_BYTES) - ret = nacl.crypto_auth_verify(tok, msg, ctypes.c_ulonglong(len(msg)), key) - if ret: - raise ValueError('Failed to auth msg') - return tok.raw[:crypto_auth_BYTES] - -# One time authentication - - -def crypto_onetimeauth(msg, key): - ''' - Constructs a one time authentication token for the given message msg using - a given secret key - ''' - tok = ctypes.create_string_buffer(crypto_onetimeauth_BYTES) - ret = nacl.crypto_onetimeauth(tok, msg, ctypes.c_ulonglong(len(msg)), key) - if ret: - raise ValueError('Failed to auth msg') - return tok.raw[:crypto_onetimeauth_BYTES] - - -def crypto_onetimeauth_verify(msg, key): - ''' - Verifies that the given authentication token is correct for the given - message and key - ''' - tok = ctypes.create_string_buffer(crypto_onetimeauth_BYTES) - ret = nacl.crypto_onetimeauth(tok, msg, ctypes.c_ulonglong(len(msg)), key) - if ret: - raise ValueError('Failed to auth msg') - return tok.raw[:crypto_onetimeauth_BYTES] - -# Hashing - - -def crypto_hash(msg): - ''' - Compute a hash of the given message - ''' - hbuf = ctypes.create_string_buffer(crypto_hash_BYTES) - nacl.crypto_hash(hbuf, msg, ctypes.c_ulonglong(len(msg))) - return hbuf.raw - - -def crypto_hash_sha256(msg): - ''' - Compute the sha256 hash of the given message - ''' - hbuf = ctypes.create_string_buffer(crypto_hash_sha256_BYTES) - nacl.crypto_hash_sha256(hbuf, msg, ctypes.c_ulonglong(len(msg))) - return hbuf.raw - - -def crypto_hash_sha512(msg): - ''' - Compute the sha512 hash of the given message - ''' - hbuf = ctypes.create_string_buffer(crypto_hash_sha512_BYTES) - nacl.crypto_hash_sha512(hbuf, msg, ctypes.c_ulonglong(len(msg))) - return hbuf.raw - -# Generic Hash - - -def crypto_generichash(msg, key=None): - ''' - Compute the blake2 hash of the given message with a given key - ''' - hbuf = ctypes.create_string_buffer(crypto_generichash_BYTES) - if key: - key_len = len(key) - else: - key_len = 0 - nacl.crypto_generichash(hbuf, ctypes.c_ulonglong(len(hbuf)), msg, - ctypes.c_ulonglong(len(msg)), key, - ctypes.c_ulonglong(key_len)) - return hbuf.raw - -# scalarmult - - -def crypto_scalarmult_base(n): - ''' - Computes and returns the scalar product of a standard group element and an - integer "n". - ''' - buf = ctypes.create_string_buffer(crypto_scalarmult_BYTES) - ret = nacl.crypto_scalarmult_base(buf, n) - if ret: - raise CryptError('Failed to compute scalar product') - return buf.raw - -# String cmp - - -def crypto_verify_16(string1, string2): - ''' - Compares the first crypto_verify_16_BYTES of the given strings - - The time taken by the function is independent of the contents of string1 - and string2. In contrast, the standard C comparison function - memcmp(string1,string2,16) takes time that is dependent on the longest - matching prefix of string1 and string2. This often allows for easy - timing attacks. - ''' - return not nacl.crypto_verify_16(string1, string2) - - -def crypto_verify_32(string1, string2): - ''' - Compares the first crypto_verify_32_BYTES of the given strings - - The time taken by the function is independent of the contents of string1 - and string2. In contrast, the standard C comparison function - memcmp(string1,string2,16) takes time that is dependent on the longest - matching prefix of string1 and string2. This often allows for easy - timing attacks. - ''' - return not nacl.crypto_verify_32(string1, string2) - -# Random byte generation - - -def randombytes(size): - ''' - Return a string of random bytes of the given size - ''' - buf = ctypes.create_string_buffer(size) - nacl.randombytes(buf, ctypes.c_ulonglong(size)) - return buf.raw - - -def randombytes_buf(size): - ''' - Return a string of random bytes of the given size - ''' - size = int(size) - buf = ctypes.create_string_buffer(size) - nacl.randombytes_buf(buf, size) - return buf.raw - - -def randombytes_close(): - ''' - Close the file descriptor or the handle for the cryptographic service - provider - ''' - nacl.randombytes_close() - - -def randombytes_random(): - ''' - Return a random 32-bit unsigned value - ''' - return nacl.randombytes_random() - - -def randombytes_stir(): - ''' - Generate a new key for the pseudorandom number generator - - The file descriptor for the entropy source is kept open, so that the - generator can be reseeded even in a chroot() jail. - ''' - nacl.randombytes_stir() - - -def randombytes_uniform(upper_bound): - ''' - Return a value between 0 and upper_bound using a uniform distribution - ''' - return nacl.randombytes_uniform(upper_bound) - -# Utility functions - - -def sodium_library_version_major(): - ''' - Return the major version number - ''' - return nacl.sodium_library_version_major() - - -def sodium_library_version_minor(): - ''' - Return the minor version number - ''' - return nacl.sodium_library_version_minor() - - -def sodium_version_string(): - ''' - Return the version string - ''' - func = nacl.sodium_version_string - func.restype = ctypes.c_char_p - return func() diff --git a/libnacl/base.py b/libnacl/base.py deleted file mode 100644 index f69a6d7e..00000000 --- a/libnacl/base.py +++ /dev/null @@ -1,66 +0,0 @@ -# -*- coding: utf-8 -*- -''' -Implement the base key object for other keys to inherit convenience functions -''' -# Import libnacl libs -import libnacl.encode - -# Import python libs -import os -import stat - - -class BaseKey(object): - ''' - Include methods for key management convenience - ''' - - def hex_sk(self): - if hasattr(self, 'sk'): - return libnacl.encode.hex_encode(self.sk) - else: - return '' - - def hex_pk(self): - if hasattr(self, 'pk'): - return libnacl.encode.hex_encode(self.pk) - - def hex_vk(self): - if hasattr(self, 'vk'): - return libnacl.encode.hex_encode(self.vk) - - def hex_seed(self): - if hasattr(self, 'seed'): - return libnacl.encode.hex_encode(self.seed) - - def save(self, path, serial='json'): - ''' - Safely save keys with perms of 0400 - ''' - pre = {} - sk = self.hex_sk() - pk = self.hex_pk() - vk = self.hex_vk() - seed = self.hex_seed() - if sk and pk: - pre['priv'] = sk.decode('utf-8') - if pk: - pre['pub'] = pk.decode('utf-8') - if vk: - pre['verify'] = vk.decode('utf-8') - if seed: - pre['sign'] = seed.decode('utf-8') - if serial == 'msgpack': - import msgpack - packaged = msgpack.dumps(pre) - elif serial == 'json': - import json - packaged = json.dumps(pre) - - perm_other = stat.S_IWOTH | stat.S_IXOTH | stat.S_IWOTH - perm_group = stat.S_IXGRP | stat.S_IWGRP | stat.S_IRWXG - - cumask = os.umask(perm_other | perm_group) - with open(path, 'w+') as fp_: - fp_.write(packaged) - os.umask(cumask) diff --git a/libnacl/blake.py b/libnacl/blake.py deleted file mode 100644 index a1016fc6..00000000 --- a/libnacl/blake.py +++ /dev/null @@ -1,45 +0,0 @@ -''' -Mimic very closely the python hashlib classes for blake2b - -NOTE: - This class does not yet implement streaming the msg into the - hash function via the update method -''' - -# Import python libs -import binascii - -# Import libnacl libs -import libnacl - - -class Blake2b(object): - ''' - Manage a Blake2b hash - ''' - - def __init__(self, msg, key=None): - self.msg = msg - self.key = key - self.raw_digest = libnacl.crypto_generichash(msg, key) - self.digest_size = len(self.raw_digest) - - def digest(self): - ''' - Return the digest of the string - ''' - return self.raw_digest - - def hexdigest(self): - ''' - Return the hex digest of the string - ''' - return binascii.hexlify(self.raw_digest) - - -def blake2b(msg, key=None): - ''' - Create and return a Blake2b object to mimic the behavior of the python - hashlib functions - ''' - return Blake2b(msg, key) diff --git a/libnacl/dual.py b/libnacl/dual.py deleted file mode 100644 index af802225..00000000 --- a/libnacl/dual.py +++ /dev/null @@ -1,35 +0,0 @@ -''' -The dual key system allows for the creation of keypairs that contain both -cryptographic and signing keys -''' -# import libnacl libs -import libnacl -import libnacl.base -import libnacl.public -import libnacl.sign - - -class DualSecret(libnacl.base.BaseKey): - ''' - Manage crypt and sign keys in one object - ''' - - def __init__(self, crypt=None, sign=None): - self.crypt = libnacl.public.SecretKey(crypt) - self.signer = libnacl.sign.Signer(sign) - self.sk = self.crypt.sk - self.seed = self.signer.seed - self.pk = self.crypt.pk - self.vk = self.signer.vk - - def sign(self, msg): - ''' - Sign the given message - ''' - return self.signer.sign(msg) - - def signature(self, msg): - ''' - Return just the signature for the message - ''' - return self.signer.signature(msg) diff --git a/libnacl/encode.py b/libnacl/encode.py deleted file mode 100644 index efbfe998..00000000 --- a/libnacl/encode.py +++ /dev/null @@ -1,63 +0,0 @@ -# -*- coding: utf-8 -*- -''' -Build in routines and classes to simplify encoding routines -''' -# Import python libs -import base64 -import binascii - - -def hex_encode(data): - ''' - Hex encode data - ''' - return binascii.hexlify(data) - - -def hex_decode(data): - ''' - Hex decode data - ''' - return binascii.unhexlify(data) - - -def base16_encode(data): - ''' - Base32 encode data - ''' - return base64.b16encode(data) - - -def base16_decode(data): - ''' - Base16 decode data - ''' - return base64.b16decode(data) - - -def base32_encode(data): - ''' - Base16 encode data - ''' - return base64.b32encode(data) - - -def base32_decode(data): - ''' - Base32 decode data - ''' - return base64.b32decode(data) - - -def base64_encode(data): - ''' - Base16 encode data - ''' - return base64.b64encode(data) - - -def base64_decode(data): - ''' - Base32 decode data - ''' - return base64.b64decode(data) diff --git a/libnacl/public.py b/libnacl/public.py deleted file mode 100644 index 6722ccf7..00000000 --- a/libnacl/public.py +++ /dev/null @@ -1,84 +0,0 @@ -# -*- coding: utf-8 -*- -''' -High level classes and routines around public key encryption and decryption -''' -# import libnacl libs -import libnacl -import libnacl.utils -import libnacl.encode -import libnacl.dual -import libnacl.base - - -class PublicKey(libnacl.base.BaseKey): - ''' - This class is used to manage public keys - ''' - - def __init__(self, pk): - self.pk = pk - - -class SecretKey(libnacl.base.BaseKey): - ''' - This class is used to manage keypairs - ''' - - def __init__(self, sk=None): - ''' - If a secret key is not passed in then it will be generated - ''' - if sk is None: - self.pk, self.sk = libnacl.crypto_box_keypair() - elif len(sk) == libnacl.crypto_box_SECRETKEYBYTES: - self.sk = sk - self.pk = libnacl.crypto_scalarmult_base(sk) - else: - raise ValueError('Passed in invalid secret key') - - -class Box(object): - ''' - TheBox class is used to create cryptographic boxes and unpack - cryptographic boxes - ''' - - def __init__(self, sk, pk): - if isinstance(sk, (SecretKey, libnacl.dual.DualSecret)): - sk = sk.sk - if isinstance(pk, (SecretKey, libnacl.dual.DualSecret)): - raise ValueError('Passed in secret key as public key') - if isinstance(pk, PublicKey): - pk = pk.pk - if pk and sk: - self._k = libnacl.crypto_box_beforenm(pk, sk) - - def encrypt(self, msg, nonce=None, pack_nonce=True): - ''' - Encrypt the given message with the given nonce, if the nonce is not - provided it will be generated from the libnacl.utils.rand_nonce - function - ''' - if nonce is None: - nonce = libnacl.utils.rand_nonce() - elif len(nonce) != libnacl.crypto_box_NONCEBYTES: - raise ValueError('Invalid nonce size') - ctxt = libnacl.crypto_box_afternm(msg, nonce, self._k) - if pack_nonce: - return nonce + ctxt - else: - return nonce, ctxt - - def decrypt(self, ctxt, nonce=None): - ''' - Decrypt the given message, if a nonce is passed in attempt to decrypt - it with the given nonce, otherwise assum that the nonce is attached - to the message - ''' - if nonce is None: - nonce = ctxt[:libnacl.crypto_box_NONCEBYTES] - ctxt = ctxt[libnacl.crypto_box_NONCEBYTES:] - elif len(nonce) != libnacl.crypto_box_NONCEBYTES: - raise ValueError('Invalid nonce') - msg = libnacl.crypto_box_open_afternm(ctxt, nonce, self._k) - return msg diff --git a/libnacl/secret.py b/libnacl/secret.py deleted file mode 100644 index ea1b8cf0..00000000 --- a/libnacl/secret.py +++ /dev/null @@ -1,45 +0,0 @@ -# -*- coding: utf-8 -*- -''' -Utilities to make secret box encryption simple -''' -# Import libnacl -import libnacl -import libnacl.utils -import libnacl.base - - -class SecretBox(libnacl.base.BaseKey): - ''' - Manage symetric encryption using the salsa20 algorithm - ''' - - def __init__(self, key=None): - if key is None: - key = libnacl.utils.salsa_key() - if len(key) != libnacl.crypto_secretbox_KEYBYTES: - raise ValueError('Invalid key') - self.sk = key - - def encrypt(self, msg, nonce=None): - ''' - Encrypt the given message. If a nonce is not given it will be - generated via the rand_nonce function - ''' - if nonce is None: - nonce = libnacl.utils.rand_nonce() - if len(nonce) != libnacl.crypto_secretbox_NONCEBYTES: - raise ValueError('Invalid Nonce') - ctxt = libnacl.crypto_secretbox(msg, nonce, self.sk) - return nonce + ctxt - - def decrypt(self, ctxt, nonce=None): - ''' - Decrypt the given message, if no nonce is given the nonce will be - extracted from the message - ''' - if nonce is None: - nonce = ctxt[:libnacl.crypto_secretbox_NONCEBYTES] - ctxt = ctxt[libnacl.crypto_secretbox_NONCEBYTES:] - if len(nonce) != libnacl.crypto_secretbox_NONCEBYTES: - raise ValueError('Invalid nonce') - return libnacl.crypto_secretbox_open(ctxt, nonce, self.sk) diff --git a/libnacl/sign.py b/libnacl/sign.py deleted file mode 100644 index 850bdf61..00000000 --- a/libnacl/sign.py +++ /dev/null @@ -1,57 +0,0 @@ -# -*- coding: utf-8 -*- -''' -High level routines to maintain signing keys and to sign and verify messages -''' -# Import libancl libs -import libnacl -import libnacl.base -import libnacl.encode - - -class Signer(libnacl.base.BaseKey): - ''' - The tools needed to sign messages - ''' - - def __init__(self, seed=None): - ''' - Create a signing key, if not seed it supplied a keypair is generated - ''' - if seed: - if len(seed) != libnacl.crypto_sign_SEEDBYTES: - raise ValueError('Invalid seed bytes') - self.vk, self.sk = libnacl.crypto_sign_seed_keypair(seed) - else: - seed = libnacl.randombytes(libnacl.crypto_sign_SEEDBYTES) - self.vk, self.sk = libnacl.crypto_sign_seed_keypair(seed) - self.seed = seed - - def sign(self, msg): - ''' - Sign the given message with this key - ''' - return libnacl.crypto_sign(msg, self.sk) - - def signature(self, msg): - ''' - Return just the signature for the message - ''' - return libnacl.crypto_sign(msg, self.sk)[:libnacl.crypto_sign_BYTES] - - -class Verifier(libnacl.base.BaseKey): - ''' - Verify signed messages - ''' - - def __init__(self, vk_hex): - ''' - Create a verification key from a hex encoded vkey - ''' - self.vk = libnacl.encode.hex_decode(vk_hex) - - def verify(self, msg): - ''' - Verify the message with tis key - ''' - return libnacl.crypto_sign_open(msg, self.vk) diff --git a/libnacl/utils.py b/libnacl/utils.py deleted file mode 100644 index 58bc1699..00000000 --- a/libnacl/utils.py +++ /dev/null @@ -1,66 +0,0 @@ -# -*- coding: utf-8 -*- - -import struct -import time - -# Import nacl libs -import libnacl -import libnacl.encode -import libnacl.public -import libnacl.sign -import libnacl.dual - - -def load_key(path, serial='json'): - ''' - Read in a key from a file and return the applicable key object based on - the contents of the file - ''' - with open(path, 'rb') as fp_: - packaged = fp_.read() - if serial == 'msgpack': - import msgpack - key_data = msgpack.loads(packaged) - elif serial == 'json': - import json - key_data = json.loads(packaged.decode(encoding='UTF-8')) - if 'priv' in key_data and 'sign' in key_data: - return libnacl.dual.DualSecret( - libnacl.encode.hex_decode(key_data['priv']), - libnacl.encode.hex_decode(key_data['sign'])) - elif 'priv' in key_data: - return libnacl.public.SecretKey(libnacl.encode.hex_decode(key_data[ - 'priv'])) - elif 'sign' in key_data: - return libnacl.sign.Signer(libnacl.encode.hex_decode(key_data['sign'])) - elif 'pub' in key_data: - return libnacl.public.PublicKey(libnacl.encode.hex_decode(key_data[ - 'pub'])) - elif 'verify' in key_data: - return libnacl.sign.Verifier(key_data['verify']) - raise ValueError('Found no key data') - - -def salsa_key(): - ''' - Generates a salsa2020 key - ''' - return libnacl.randombytes(libnacl.crypto_secretbox_KEYBYTES) - - -def rand_nonce(): - ''' - Generates and returns a random bytestring of the size defined in libsodium - as crypto_box_NONCEBYTES - ''' - return libnacl.randombytes(libnacl.crypto_box_NONCEBYTES) - - -def time_nonce(): - ''' - Generates and returns a nonce as in rand_nonce() but using a timestamp for the first 8 bytes. - - This function now exists mostly for backwards compatibility, as rand_nonce() is usually preferred. - ''' - nonce = rand_nonce() - return (struct.pack('=d', time.time()) + nonce)[:len(nonce)] diff --git a/libnacl/version.py b/libnacl/version.py deleted file mode 100644 index 96e3ce8d..00000000 --- a/libnacl/version.py +++ /dev/null @@ -1 +0,0 @@ -__version__ = '1.4.0' diff --git a/miniircd b/miniircd deleted file mode 160000 index 49dce868..00000000 --- a/miniircd +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 49dce8686be06cd04d8c9fd97f1eccf09984d062 diff --git a/ob-watcher.py b/ob-watcher.py index c3dcab49..431fec39 100644 --- a/ob-watcher.py +++ b/ob-watcher.py @@ -18,8 +18,9 @@ # https://stackoverflow.com/questions/2801882/generating-a-png-with-matplotlib-when-display-is-undefined import matplotlib -from joinmarket import jm_single, load_program_config, IRCMessageChannel -from joinmarket import random_nick, calc_cj_fee, OrderbookWatch +from joinmarket import jm_single, load_program_config, MessageChannelCollection +from joinmarket import random_nick, calc_cj_fee, OrderbookWatch, get_irc_mchannels +from joinmarket import IRCMessageChannel matplotlib.use('Agg') import matplotlib.pyplot as plt @@ -112,14 +113,14 @@ def do_nothing(arg, order, btc_unit, rel_unit): def ordertype_display(ordertype, order, btc_unit, rel_unit): - ordertypes = {'absorder': 'Absolute Fee', 'relorder': 'Relative Fee'} + ordertypes = {'absoffer': 'Absolute Fee', 'reloffer': 'Relative Fee'} return ordertypes[ordertype] def cjfee_display(cjfee, order, btc_unit, rel_unit): - if order['ordertype'] == 'absorder': + if order['ordertype'] == 'absoffer': return satoshi_to_unit(cjfee, order, btc_unit, rel_unit) - elif order['ordertype'] == 'relorder': + elif order['ordertype'] == 'reloffer': return str(float(cjfee) * rel_unit_to_factor[rel_unit]) + rel_unit @@ -144,7 +145,7 @@ def create_orderbook_table(db, btc_unit, rel_unit): ('minsize', satoshi_to_unit), ('maxsize', satoshi_to_unit)) - # somewhat complex sorting to sort by cjfee but with absorders on top + # somewhat complex sorting to sort by cjfee but with absoffers on top def orderby_cmp(x, y): if x['ordertype'] == y['ordertype']: @@ -211,7 +212,7 @@ def create_orderbook_obj(self): o = dict(row) if 'cjfee' in o: o['cjfee'] = int(o['cjfee']) if o[ - 'ordertype'] == 'absorder' else float( + 'ordertype'] == 'absoffer' else float( o['cjfee']) result.append(o) return result @@ -366,14 +367,14 @@ def main(): (options, args) = parser.parse_args() hostport = (options.host, options.port) - - irc = IRCMessageChannel(jm_single().nickname) + mcs = [IRCMessageChannel(c) for c in get_irc_mchannels()] + mcc = MessageChannelCollection(mcs) # todo: is the call to GUITaker needed, or the return. taker unused - taker = GUITaker(irc, hostport) + taker = GUITaker(mcc, hostport) print('starting irc') - irc.run() + mcc.run() if __name__ == "__main__": diff --git a/patientsendpayment.py b/patientsendpayment.py index 41ce1d24..c73fa922 100644 --- a/patientsendpayment.py +++ b/patientsendpayment.py @@ -85,7 +85,7 @@ def create_my_orders(self): # choose an absolute fee order to discourage people from # mixing smaller amounts order = {'oid': 0, - 'ordertype': 'absorder', + 'ordertype': 'absoffer', 'minsize': 0, 'maxsize': self.amount, 'txfee': self.txfee, @@ -110,7 +110,7 @@ def on_tx_unconfirmed(self, cjorder, balance, removed_utxos): available_balance = self.wallet.get_balance_by_mixdepth()[self.mixdepth] if available_balance >= self.amount: order = {'oid': 0, - 'ordertype': 'absorder', + 'ordertype': 'absoffer', 'minsize': 0, 'maxsize': self.amount, 'txfee': self.txfee, @@ -123,7 +123,7 @@ def on_tx_unconfirmed(self, cjorder, balance, removed_utxos): def on_tx_confirmed(self, cjorder, confirmations, txid, balance): if len(self.orderlist) == 0: order = {'oid': 0, - 'ordertype': 'absorder', + 'ordertype': 'absoffer', 'minsize': 0, 'maxsize': self.amount, 'txfee': self.txfee, diff --git a/requirements-dev.txt b/requirements-dev.txt index befc9759..915938f1 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -1,8 +1,8 @@ # matplotlib # numpy pexpect -pytest -pytest-cov +pytest==2.8.2 +pytest-cov==2.2.1 python-coveralls -r requirements.txt diff --git a/requirements-windows.txt b/requirements-windows.txt new file mode 100644 index 00000000..08a71b66 --- /dev/null +++ b/requirements-windows.txt @@ -0,0 +1,2 @@ +libnacl>=1.0.4 +secp256k1-transient \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index e69de29b..9323b99e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -0,0 +1,2 @@ +libnacl>=1.0.4 +secp256k1>=0.13.1 \ No newline at end of file diff --git a/sendpayment.py b/sendpayment.py index cd7a9394..52e6c844 100644 --- a/sendpayment.py +++ b/sendpayment.py @@ -10,9 +10,9 @@ # sys.path.insert(0, os.path.join(data_dir, 'joinmarket')) import time -from joinmarket import Taker, load_program_config, IRCMessageChannel +from joinmarket import Taker, load_program_config, IRCMessageChannel, \ + MessageChannelCollection, get_irc_mchannels from joinmarket import validate_address, jm_single -from joinmarket import random_nick from joinmarket import get_log, choose_sweep_orders, choose_orders, \ pick_order, cheapest_order_choose, weighted_order_choose, debug_dump_object from joinmarket import Wallet, BitcoinCoreWallet @@ -249,6 +249,13 @@ def main(): dest='mixdepth', help='mixing depth to spend from, default=0', default=0) + parser.add_option('-a', + '--amtmixdepths', + action='store', + type='int', + dest='amtmixdepths', + help='number of mixdepths in wallet, default 5', + default=5) parser.add_option('-g', '--gap-limit', type="int", @@ -303,23 +310,22 @@ def main(): log.debug("Estimated miner/tx fee for each cj participant: "+str(options.txfee)) assert(options.txfee >= 0) - jm_single().nickname = random_nick() - log.debug('starting sendpayment') if not options.userpcwallet: - wallet = Wallet(wallet_name, options.mixdepth + 1, options.gaplimit) + wallet = Wallet(wallet_name, options.amtmixdepths, options.gaplimit) else: wallet = BitcoinCoreWallet(fromaccount=wallet_name) jm_single().bc_interface.sync_wallet(wallet) - irc = IRCMessageChannel(jm_single().nickname) - taker = SendPayment(irc, wallet, destaddr, amount, options.makercount, + mcs = [IRCMessageChannel(c) for c in get_irc_mchannels()] + mcc = MessageChannelCollection(mcs) + taker = SendPayment(mcc, wallet, destaddr, amount, options.makercount, options.txfee, options.waittime, options.mixdepth, options.answeryes, chooseOrdersFunc) try: - log.debug('starting irc') - irc.run() + log.debug('starting message channels') + mcc.run() except: log.debug('CRASHING, DUMPING EVERYTHING') debug_dump_object(wallet, ['addr_cache', 'keys', 'wallet_name', 'seed']) diff --git a/setup.cfg b/setup.cfg deleted file mode 100644 index af254176..00000000 --- a/setup.cfg +++ /dev/null @@ -1,11 +0,0 @@ -[flake8] -exclude = env* -max-line-length = 150 - -[pep8] -exclude = env*,.tox -max-line-length = 150 - -[pytest] -addopts = --cov=joinmarket/ --cov=test/ --cov=bitcoin/ -norecursedirs = .tox env diff --git a/test/conftest.py b/test/conftest.py index 9dea36f4..78dd9afd 100644 --- a/test/conftest.py +++ b/test/conftest.py @@ -9,7 +9,7 @@ bitcoin_conf = None bitcoin_rpcpassword = None bitcoin_rpcusername = None -miniircd_proc = None +miniircd_procs = [] def pytest_addoption(parser): @@ -27,11 +27,17 @@ def pytest_addoption(parser): action="store", default='bitcoinrpc', help="the RPC username for your test bitcoin instance (default=bitcoinrpc)") + parser.addoption("--nirc", + type="int", + action="store", + default=1, + help="the number of local miniircd instances") def teardown(): #didn't find a stop command in miniircd, so just kill - global miniircd_proc - miniircd_proc.kill() + global miniircd_procs + for m in miniircd_procs: + m.kill() #shut down bitcoin and remove the regtest dir local_command([bitcoin_path + "bitcoin-cli", "-regtest", "-rpcuser=" + bitcoin_rpcusername, @@ -53,10 +59,14 @@ def setup(request): #start up miniircd #minor bug in miniircd (seems); need *full* unqualified path for motd file cwd = os.getcwd() - global miniircd_proc - miniircd_proc = local_command( - ["./miniircd/miniircd", "--motd=" + cwd + "/miniircd/testmotd"], - bg=True) + n_irc = request.config.getoption("--nirc") + global miniircd_procs + for i in range(n_irc): + miniircd_proc = local_command( + ["./miniircd/miniircd", "--ports=" + str(6667+i), + "--motd=" + cwd + "/miniircd/testmotd"], + bg=True) + miniircd_procs.append(miniircd_proc) #start up regtest blockchain btc_proc = subprocess.call([bitcoin_path + "bitcoind", "-regtest", "-daemon", "-conf=" + bitcoin_conf]) diff --git a/test/regtest_joinmarket.cfg b/test/regtest_joinmarket.cfg index 22ff8d6f..50ba79d7 100644 --- a/test/regtest_joinmarket.cfg +++ b/test/regtest_joinmarket.cfg @@ -10,13 +10,14 @@ network = testnet bitcoin_cli_cmd = bitcoin-cli notify_port = 62612 [MESSAGING] -host = localhost -channel = joinmarket-pit -port = 6667 -usessl = false -socks5 = false -socks5_host = localhost -socks5_port = 9150 +host = localhost, localhost +hostid = localhost1, localhost2 +channel = joinmarket-pit, joinmarket-pit +port = 6667, 6668 +usessl = false, false +socks5 = false, false +socks5_host = localhost, localhost +socks5_port = 9150, 9150 [POLICY] # for dust sweeping, try merge_algorithm = gradual # for more rapid dust sweeping, try merge_algorithm = greedy @@ -30,4 +31,7 @@ merge_algorithm = default # as our default. Note that for clients not using a local blockchain # instance, we retrieve an estimate from the API at cointape.com, currently. tx_fees = 3 - +taker_utxo_retries = 3 +taker_utxo_age = 1 +taker_utxo_amtpercent = 20 +accept_commitment_broadcasts = 1 diff --git a/test/test_blockr.py b/test/test_blockr.py index b0201b1b..c7f5f6a9 100644 --- a/test/test_blockr.py +++ b/test/test_blockr.py @@ -84,12 +84,8 @@ def cus_print(s): balance_depth += balance used = ('used' if k < wallet.index[m][forchange] else ' new') if showprivkey: - if btc.secp_present: - privkey = btc.wif_compressed_privkey( + privkey = btc.wif_compressed_privkey( wallet.get_key(m, forchange, k), get_p2pk_vbyte()) - else: - privkey = btc.encode_privkey(wallet.get_key(m, - forchange, k), 'wif_compressed', get_p2pk_vbyte()) else: privkey = '' if (method == 'displayall' or balance > 0 or diff --git a/test/test_broadcast_method.py b/test/test_broadcast_method.py index bc1d912d..10832913 100644 --- a/test/test_broadcast_method.py +++ b/test/test_broadcast_method.py @@ -36,8 +36,10 @@ def shutdown(self): pass def _pubmsg(self, msg): pass def _privmsg(self, nick, cmd, message): pass def _announce_orders(self, orderlist, nick): pass - - def fill_orders(self, nick_order_dict, cj_amount, taker_pubkey): pass + def change_nick(self, new_nick): + self.nick = new_nick + def fill_orders(self, nick_order_dict, cj_amount, taker_pubkey, commitment): + pass def push_tx(self, nick, txhex): msgchan_pushtx_count[0][nick] += 1 @@ -55,6 +57,9 @@ def pushtx(self, txhex): self_pushtx_count[0] += 1 return True +def dummy_commitment_creator(wallet, utxos, amount): + return "fake_commitment", "fake_reveal" + def create_testing_cjtx(counterparty_list): msgchan_pushtx_count[0] = dict(zip(counterparty_list, [0]* len(counterparty_list))) @@ -63,7 +68,7 @@ def create_testing_cjtx(counterparty_list): input_utxos = {'utxo-hex-here': {'value': 0, 'address': '1addr'}} cjtx = CoinJoinTX(DummyMessageChannel(), None, None, 0, orders, input_utxos - , '1cjaddr', '1changeaddr', 0, None, None, None) + , '1cjaddr', '1changeaddr', 0, None, None, dummy_commitment_creator) cjtx.latest_tx = btc.deserialize(sample_tx_hex) return cjtx @@ -108,7 +113,7 @@ def test_broadcast_random_maker(setup_tx_notify): for m in makers: cjtx.db.execute( 'INSERT INTO orderbook VALUES(?, ?, ?, ?, ?, ?, ?);', - (m, 0, 'absorder', 0, 2100000000000001, 1000, 1000)) + (m, 0, 'absoffer', 0, 2100000000000001, 1000, 1000)) msgchan_pushtx_count[0] = dict(zip(makers, [0]*len(makers))) N = 1000 for i in xrange(N): diff --git a/test/test_donations.py b/test/test_donations.py index e0986c6d..95543e45 100644 --- a/test/test_donations.py +++ b/test/test_donations.py @@ -1,109 +1,96 @@ -import sys +#! /usr/bin/env python +from __future__ import absolute_import +'''Tests of reusable donation pubkey code using secp256k1 ''' + +import binascii +import time import bitcoin as btc +import secp256k1 import pytest -import json + from commontest import make_wallets -from joinmarket import load_program_config, get_p2pk_vbyte, get_log +from joinmarket import load_program_config, get_p2pk_vbyte, get_log, jm_single +from joinmarket import get_irc_mchannels, Taker +from joinmarket.configure import donation_address log = get_log() -# This is a primitive version -# to test the basic concept is working (specifically the imports). -# It isn't required that the transaction is valid, only that a valid -# utxo from the wallet can extract a private key. -# A real test of donations fits into the tumbler test cases, although -# with effort an "intermediately realistic" version could be set up. - - @pytest.mark.parametrize( - "tx_type, tx_id, tx_hex, address", - [("simple-tx", - "f31916a1d398a4ec18d56a311c942bb6db934cee6aa8ac30af0b30aad9efb841", - "0100000001c74265f31fc5e24895fdc83f7157cc40045235f3a71ae326a219de9de873" + - "0d8b010000006a473044022076055917470b7ec4f4bb008096266cf816ebb089ad983e" + - "6a0f63340ba0e6a6cb022059ec938b996a75db10504e46830e13d399f28191b9832bd5" + - "f61df097b9e0d47801210291941334a00959af4aa5757abf81d2a7d1aca8adb3431c67" + - "e89419271ba71cb4feffffff023cdeeb03000000001976a914a2426748f14eba44b3f6" + - "abba3e8bce216ea233f388acf4ebf303000000001976a914bfa366464a464005ba0df8" + - "6024a6c3ed859f03ac88ac33280600", - "msWrR3Gm2mBmdLZH8vGHbHifM53N2vuYBq"), - + "amount", + [(100000000 + ), ]) -def test_donation_address(setup_donations, tx_type, tx_id, tx_hex, address): +def test_donation_address(setup_donations, amount): wallets = make_wallets(1, wallet_structures=[[1,1,1,0,0]], mean_amt=0.5) wallet = wallets[0]['wallet'] - priv, addr = donation_address(tx_hex, wallet) - print addr - #just a check that it doesn't throw - sign_donation_tx(tx_hex, 0, priv) + jm_single().bc_interface.sync_wallet(wallet) + #make a rdp from a simple privkey + rdp_priv = "\x01"*32 + reusable_donation_pubkey = binascii.hexlify(secp256k1.PrivateKey( + privkey=rdp_priv, raw=True, ctx=btc.ctx).pubkey.serialize()) + dest_addr, sign_k = donation_address(reusable_donation_pubkey) + print dest_addr + jm_single().bc_interface.rpc('importaddress', + [dest_addr, '', False]) + ins_full = wallet.unspent + total = sum(x['value'] for x in ins_full.values()) + ins = ins_full.keys() + output_addr = wallet.get_new_addr(1, 1) + fee_est = 10000 + outs = [{'value': amount, + 'address': dest_addr}, {'value': total - amount - fee_est, + 'address': output_addr}] -if not btc.secp_present: - #See note above, this is NOT the real code, see taker.py - def donation_address(tx, wallet): - from bitcoin.main import multiply, G, deterministic_generate_k, add_pubkeys - reusable_donation_pubkey = ('02be838257fbfddabaea03afbb9f16e852' - '9dfe2de921260a5c46036d97b5eacf2a') - - privkey = wallet.get_key_from_addr(wallet.get_new_addr(0,0)) - msghash = btc.bin_txhash(tx, btc.SIGHASH_ALL) - # generate unpredictable k - global sign_k - sign_k = deterministic_generate_k(msghash, privkey) - c = btc.sha256(multiply(reusable_donation_pubkey, sign_k)) - sender_pubkey = add_pubkeys( - reusable_donation_pubkey, multiply( - G, c)) - sender_address = btc.pubtoaddr(sender_pubkey, get_p2pk_vbyte()) - log.debug('sending coins to ' + sender_address) - return privkey, sender_address - - #See note above, this is NOT the real code, see taker.py - def sign_donation_tx(tx, i, priv): - from bitcoin.main import fast_multiply, decode_privkey, G, inv, N - from bitcoin.transaction import der_encode_sig - k = sign_k - hashcode = btc.SIGHASH_ALL - i = int(i) - if len(priv) <= 33: - priv = btc.safe_hexlify(priv) - pub = btc.privkey_to_pubkey(priv) - address = btc.pubkey_to_address(pub) - signing_tx = btc.signature_form( - tx, i, btc.mk_pubkey_script(address), hashcode) - - msghash = btc.bin_txhash(signing_tx, hashcode) - z = btc.hash_to_int(msghash) - # k = deterministic_generate_k(msghash, priv) - r, y = fast_multiply(G, k) - s = inv(k, N) * (z + r * decode_privkey(priv)) % N - rawsig = 27 + (y % 2), r, s - - sig = der_encode_sig(*rawsig) + btc.encode(hashcode, 16, 2) - # sig = ecdsa_tx_sign(signing_tx, priv, hashcode) - txobj = btc.deserialize(tx) - txobj["ins"][i]["script"] = btc.serialize_script([sig, pub]) - return btc.serialize(txobj) -else: - def donation_address(cjtx, wallet): - privkey = wallet.get_key_from_addr(wallet.get_new_addr(0,0)) - reusable_donation_pubkey = '02be838257fbfddabaea03afbb9f16e8529dfe2de921260a5c46036d97b5eacf2a' - global sign_k - import os - import binascii - sign_k = os.urandom(32) - log.debug("Using the following nonce value: "+binascii.hexlify(sign_k)) - c = btc.sha256(btc.multiply(binascii.hexlify(sign_k), - reusable_donation_pubkey, True)) - sender_pubkey = btc.add_pubkeys([reusable_donation_pubkey, - btc.privtopub(c+'01', True)], True) - sender_address = btc.pubtoaddr(sender_pubkey, get_p2pk_vbyte()) - log.debug('sending coins to ' + sender_address) - return privkey, sender_address + tx = btc.mktx(ins, outs) + de_tx = btc.deserialize(tx) + for index, ins in enumerate(de_tx['ins']): + utxo = ins['outpoint']['hash'] + ':' + str(ins['outpoint']['index']) + addr = ins_full[utxo]['address'] + priv = wallet.get_key_from_addr(addr) + priv = binascii.unhexlify(priv) + usenonce = binascii.unhexlify(sign_k) if index == 0 else None + if index == 0: + log.debug("Applying rdp to input: " + str(ins)) + tx = btc.sign(tx, index, priv, usenonce=usenonce) + #pushtx returns False on any error + push_succeed = jm_single().bc_interface.pushtx(tx) + if push_succeed: + log.debug(btc.txhash(tx)) + else: + assert False + #Role of receiver: regenerate the destination private key, + #and address, from the nonce of the first input; check it has + #received the coins. + detx = btc.deserialize(tx) + first_utxo_script = detx['ins'][0]['script'] + sig, pub = btc.deserialize_script(first_utxo_script) + log.debug(sig) + sig = binascii.unhexlify(sig) + kGlen = ord(sig[3]) + kG = sig[4:4+kGlen] + log.debug(binascii.hexlify(kG)) + if kG[0] == "\x00": + kG = kG[1:] + #H(rdp private key * K) + rdp should be ==> dest addr + #Open issue: re-introduce recovery without ECC shenanigans + #Just cheat by trying both signs for pubkey + coerced_kG_1 = "02" + binascii.hexlify(kG) + coerced_kG_2 = "03" + binascii.hexlify(kG) + for coerc in [coerced_kG_1, coerced_kG_2]: + c = btc.sha256(btc.multiply(binascii.hexlify(rdp_priv), coerc, True)) + pub_check = btc.add_pubkeys([reusable_donation_pubkey, + btc.privtopub(c+'01', True)], True) + addr_check = btc.pubtoaddr(pub_check, get_p2pk_vbyte()) + log.debug("Found checked address: " + addr_check) + if addr_check == dest_addr: + time.sleep(3) + received = jm_single().bc_interface.get_received_by_addr( + [dest_addr], None)['data'][0]['balance'] + assert received == amount + return + assert False - def sign_donation_tx(tx, i, priv): - return btc.sign(tx, i, priv, usenonce=sign_k) - @pytest.fixture(scope="module") def setup_donations(): load_program_config() diff --git a/test/test_ecc_signing.py b/test/test_ecc_signing.py index c063e083..7f3445e0 100644 --- a/test/test_ecc_signing.py +++ b/test/test_ecc_signing.py @@ -37,41 +37,6 @@ def test_valid_sigs(setup_ecc): continue assert res==False -def test_legacy_conversions(setup_ecc): - #run some checks of legacy conversion - for v in vectors['vectors']: - msg = v['msg'] - sig = binascii.unhexlify(v['sig'])[:-1] - priv = v['privkey'] - #check back-and-forth translation - assert btc.legacy_ecdsa_verify_convert( - btc.legacy_ecdsa_sign_convert(sig)) == sig - - #Correct cases passed, now try invalid signatures - - #screw up r-length - bad_sig = sig[:3] + '\x90' + sig[4:] - with pytest.raises(Exception) as e_info: - fake_sig = btc.legacy_ecdsa_sign_convert(bad_sig) - #screw up s-length - rlen = ord(sig[3]) - bad_sig = sig[:4+rlen+1] + '\x90' + sig[4+rlen+2:] - with pytest.raises(Exception) as e_info: - fake_sig = btc.legacy_ecdsa_sign_convert(bad_sig) - #valid length, but doesn't match s - bad_sig = sig[:4+rlen+1] + '\x06' + sig[4+rlen+2:] - with pytest.raises(Exception) as e_info: - fake_sig = btc.legacy_ecdsa_sign_convert(bad_sig) - #invalid inputs to legacy convert - #too short - bad_sig = '\x07'*32 - assert not btc.legacy_ecdsa_verify_convert(bad_sig) - #r OK, s too short - bad_sig = '\x07'*64 - assert not btc.legacy_ecdsa_verify_convert(bad_sig) - #note - no parity byte check, we don't bother (this is legacy) - - @pytest.fixture(scope='module') def setup_ecc(): global vectors diff --git a/test/test_irc_messaging.py b/test/test_irc_messaging.py index 63e81e54..6a9b9d73 100644 --- a/test/test_irc_messaging.py +++ b/test/test_irc_messaging.py @@ -8,20 +8,38 @@ import pytest import time import threading +import hashlib +import bitcoin as btc from commontest import local_command, make_wallets -from joinmarket.message_channel import CJPeerError -import joinmarket.irc -from joinmarket import load_program_config, IRCMessageChannel +from joinmarket.message_channel import CJPeerError, JOINMARKET_NICK_HEADER, \ + NICK_MAX_ENCODED, NICK_HASH_LENGTH +from joinmarket.irc import PING_INTERVAL +from joinmarket import load_program_config, IRCMessageChannel, get_irc_mchannels, \ + jm_single python_cmd = "python2" yg_cmd = "yield-generator-basic.py" yg_name = None class DummyMC(IRCMessageChannel): - def __init__(self, nick): - super(DummyMC, self).__init__(nick) + def __init__(self, configdata, nick): + super(DummyMC, self).__init__(configdata) + #hacked in here to allow auth without mc-collection + nick_priv = hashlib.sha256(os.urandom(16)).hexdigest() + '01' + nick_pubkey = btc.privtopub(nick_priv) + nick_pkh_raw = hashlib.sha256(nick_pubkey).digest()[ + :NICK_HASH_LENGTH] + nick_pkh = btc.changebase(nick_pkh_raw, 256, 58) + #right pad to maximum possible; b58 is not fixed length. + #Use 'O' as one of the 4 not included chars in base58. + nick_pkh += 'O' * (NICK_MAX_ENCODED - len(nick_pkh)) + #The constructed length will be 1 + 1 + NICK_MAX_ENCODED + nick = JOINMARKET_NICK_HEADER + str( + jm_single().JM_VERSION) + nick_pkh + jm_single().nickname = nick + self.set_nick(nick, nick_priv, nick_pubkey) -def on_order_seen(counterparty, oid, ordertype, minsize, +def on_order_seen(dummy, counterparty, oid, ordertype, minsize, maxsize, txfee, cjfee): global yg_name yg_name = counterparty @@ -51,7 +69,7 @@ def test_junk_messages(setup_messaging): #time.sleep(90) #start a raw IRCMessageChannel instance in a thread; #then call send_* on it with various errant messages - mc = DummyMC("irc_ping_test") + mc = DummyMC(get_irc_mchannels()[0], "irc_ping_test") mc.register_orderbookwatch_callbacks(on_order_seen=on_order_seen) mc.register_taker_callbacks(on_pubkey=on_pubkey) RawIRCThread(mc).start() @@ -77,7 +95,7 @@ def test_junk_messages(setup_messaging): #because we don't want to build a real orderbook, #call the underlying IRC announce function. #TODO: how to test that the sent format was correct? - mc._announce_orders(["!abc def gh 0001"]*30, None) + mc._announce_orders(["!abc def gh 0001"]*30) time.sleep(5) #send a fill with an invalid pubkey to the existing yg; #this should trigger a NaclError but should NOT kill it. @@ -85,12 +103,8 @@ def test_junk_messages(setup_messaging): #Test that null privmsg does not cause crash; TODO check maker log? mc.send_raw("PRIVMSG " + yg_name + " :") time.sleep(1) - #try: with pytest.raises(CJPeerError) as e_info: mc.send_error(yg_name, "fly you fools!") - #except CJPeerError: - # print "CJPeerError raised" - # pass time.sleep(5) mc.shutdown() ygp.send_signal(signal.SIGINT) @@ -99,7 +113,7 @@ def test_junk_messages(setup_messaging): @pytest.fixture(scope="module") def setup_messaging(): #Trigger PING LAG sending artificially - joinmarket.irc.PING_INTERVAL = 3 + PING_INTERVAL = 3 load_program_config() diff --git a/test/test_keys.py b/test/test_keys.py index 2dcb3f34..c56e1751 100644 --- a/test/test_keys.py +++ b/test/test_keys.py @@ -11,9 +11,6 @@ def test_read_raw_privkeys(): - if not btc.secp_present: - #these are tests of format which the pybtc library didnt do - return badkeys = ['', '\x07'*31,'\x07'*34, '\x07'*33] for b in badkeys: with pytest.raises(Exception) as e_info: diff --git a/test/test_podle.py b/test/test_podle.py new file mode 100644 index 00000000..02b3679f --- /dev/null +++ b/test/test_podle.py @@ -0,0 +1,435 @@ +#! /usr/bin/env python +from __future__ import absolute_import +'''Tests of Proof of discrete log equivalence commitments.''' +import os +import secp256k1 +import bitcoin as btc +import binascii +import json +import pytest + +import subprocess +import signal +from commontest import local_command, make_wallets +import shutil +import time +from pprint import pformat +from joinmarket import Taker, load_program_config, IRCMessageChannel +from joinmarket import validate_address, jm_single, get_irc_mchannels +from joinmarket import random_nick, get_p2pk_vbyte, MessageChannelCollection +from joinmarket import get_log, choose_sweep_orders, choose_orders, \ + pick_order, cheapest_order_choose, weighted_order_choose, debug_dump_object +import joinmarket.irc +import sendpayment + + +#for running bots as subprocesses +python_cmd = 'python2' +yg_cmd = 'yield-generator-basic.py' +#yg_cmd = 'yield-generator-mixdepth.py' +#yg_cmd = 'yield-generator-deluxe.py' + +log = get_log() + +def test_commitments_empty(setup_podle): + """Ensure that empty commitments file + results in {} + """ + assert btc.get_podle_commitments() == ([], {}) + +def test_commitment_retries(setup_podle): + """Assumes no external commitments available. + Generate pretend priv/utxo pairs and check that they can be used + taker_utxo_retries times. + """ + allowed = jm_single().config.getint("POLICY", "taker_utxo_retries") + #make some pretend commitments + dummy_priv_utxo_pairs = [(btc.sha256(os.urandom(10)), + btc.sha256(os.urandom(10))+":0") for _ in range(10)] + #test a single commitment request of all 10 + for x in dummy_priv_utxo_pairs: + p = btc.generate_podle([x], allowed) + assert p + #At this point slot 0 has been taken by all 10. + for i in range(allowed-1): + p = btc.generate_podle(dummy_priv_utxo_pairs[:1], allowed) + assert p + p = btc.generate_podle(dummy_priv_utxo_pairs[:1], allowed) + assert p is None + +def generate_single_podle_sig(priv, i): + """Make a podle entry for key priv at index i, using a dummy utxo value. + This calls the underlying 'raw' code based on the class PoDLE, not the + library 'generate_podle' which intelligently searches and updates commitments. + """ + dummy_utxo = btc.sha256(priv) + ":3" + podle = btc.PoDLE(dummy_utxo, binascii.hexlify(priv)) + r = podle.generate_podle(i) + return (r['P'], r['P2'], r['sig'], + r['e'], r['commit']) + +def test_rand_commitments(setup_podle): + #TODO bottleneck i believe is tweak_mul due + #to incorrect lack of precomputed context in upstream library. + for i in range(20): + priv = os.urandom(32) + Pser, P2ser, s, e, commitment = generate_single_podle_sig(priv, 1 + i%5) + assert btc.verify_podle(Pser, P2ser, s, e, commitment) + #tweak commitments to verify failure + tweaked = [x[::-1] for x in [Pser, P2ser, s, e, commitment]] + for i in range(5): + #Check failure on garbling of each parameter + y = [Pser, P2ser, s, e, commitment] + y[i] = tweaked[i] + fail = False + try: + fail = btc.verify_podle(*y) + except: + pass + finally: + assert not fail + +def test_nums_verify(setup_podle): + """Check that the NUMS precomputed values are + valid according to the code; assertion check + implicit. + """ + btc.verify_all_NUMS() + +@pytest.mark.parametrize( + "num_ygs, wallet_structures, mean_amt, mixdepth, sending_amt", + [ + (3, [[1, 0, 0, 0, 0]] * 4, 10, 0, 100000000), + ]) +def test_failed_sendpayment(setup_podle, num_ygs, wallet_structures, mean_amt, + mixdepth, sending_amt): + """Test of initiating joins, but failing to complete, + to see commitment usage. YGs in background as per test_regtest. + Use sweeps to avoid recover_from_nonrespondants without intruding + into sendpayment code. + """ + makercount = num_ygs + answeryes = True + txfee = 5000 + waittime = 3 + #Don't want to wait too long, but must account for possible + #throttling with !auth + jm_single().maker_timeout_sec = 12 + amount = 0 + wallets = make_wallets(makercount + 1, + wallet_structures=wallet_structures, + mean_amt=mean_amt) + #the sendpayment bot uses the last wallet in the list + wallet = wallets[makercount]['wallet'] + + yigen_procs = [] + for i in range(makercount): + ygp = local_command([python_cmd, yg_cmd,\ + str(wallets[i]['seed'])], bg=True) + time.sleep(2) #give it a chance + yigen_procs.append(ygp) + + #A significant delay is needed to wait for the yield generators to sync + time.sleep(20) + destaddr = btc.privkey_to_address( + os.urandom(32), + from_hex=False, + magicbyte=get_p2pk_vbyte()) + + addr_valid, errormsg = validate_address(destaddr) + assert addr_valid, "Invalid destination address: " + destaddr + \ + ", error message: " + errormsg + + #TODO paramatetrize this as a test variable + chooseOrdersFunc = weighted_order_choose + + log.debug('starting sendpayment') + + jm_single().bc_interface.sync_wallet(wallet) + + #Trigger PING LAG sending artificially + joinmarket.irc.PING_INTERVAL = 3 + + mcs = [IRCMessageChannel(c) for c in get_irc_mchannels()] + mcc = MessageChannelCollection(mcs) + + #Allow taker more retries than makers allow, so as to trigger + #blacklist failure case + jm_single().config.set("POLICY", "taker_utxo_retries", "4") + #override ioauth receipt with a dummy do-nothing callback: + def on_ioauth(*args): + log.debug("Taker received: " + ','.join([str(x) for x in args])) + + class DummySendPayment(sendpayment.SendPayment): + def __init__(self, msgchan, wallet, destaddr, amount, makercount, txfee, + waittime, mixdepth, answeryes, chooseOrdersFunc, on_ioauth): + self.on_ioauth = on_ioauth + self.podle_fails = 0 + self.podle_allowed_fails = 3 #arbitrary; but do it more than once + self.retries = 0 + super(DummySendPayment, self).__init__(msgchan, wallet, + destaddr, amount, makercount, txfee, waittime, + mixdepth, answeryes, chooseOrdersFunc) + def on_welcome(self): + Taker.on_welcome(self) + DummyPaymentThread(self).start() + + class DummyPaymentThread(sendpayment.PaymentThread): + def finishcallback(self, coinjointx): + #Don't ignore makers and just re-start + self.taker.retries += 1 + if self.taker.podle_fails == self.taker.podle_allowed_fails: + self.taker.msgchan.shutdown() + return + self.create_tx() + def create_tx(self): + try: + super(DummyPaymentThread, self).create_tx() + except btc.PoDLEError: + log.debug("Got one commit failure, continuing") + self.taker.podle_fails += 1 + + taker = DummySendPayment(mcc, wallet, destaddr, amount, makercount, + txfee, waittime, mixdepth, answeryes, + chooseOrdersFunc, on_ioauth) + try: + log.debug('starting message channels') + mcc.run() + finally: + if any(yigen_procs): + for ygp in yigen_procs: + #NB *GENTLE* shutdown is essential for + #test coverage reporting! + ygp.send_signal(signal.SIGINT) + ygp.wait() + #We should have been able to try (tur -1) + podle_allowed_fails times + assert taker.retries == jm_single().config.getint( + "POLICY", "taker_utxo_retries") + taker.podle_allowed_fails + #wait for block generation + time.sleep(2) + received = jm_single().bc_interface.get_received_by_addr( + [destaddr], None)['data'][0]['balance'] + #Sanity check no transaction succeeded + assert received == 0 + +def test_external_commitment_used(setup_podle): + tries = jm_single().config.getint("POLICY","taker_utxo_retries") + #Don't want to wait too long, but must account for possible + #throttling with !auth + jm_single().maker_timeout_sec = 12 + amount = 50000000 + wallets = make_wallets(3, + wallet_structures=[[1,0,0,0,0],[1,0,0,0,0],[1,1,0,0,0]], + mean_amt=1) + #the sendpayment bot uses the last wallet in the list + wallet = wallets[2]['wallet'] + yigen_procs = [] + for i in range(2): + ygp = local_command([python_cmd, yg_cmd,\ + str(wallets[i]['seed'])], bg=True) + time.sleep(2) #give it a chance + yigen_procs.append(ygp) + + #A significant delay is needed to wait for the yield generators to sync + time.sleep(10) + destaddr = btc.privkey_to_address( + binascii.hexlify(os.urandom(32)), + magicbyte=get_p2pk_vbyte()) + addr_valid, errormsg = validate_address(destaddr) + assert addr_valid, "Invalid destination address: " + destaddr + \ + ", error message: " + errormsg + + log.debug('starting sendpayment') + + jm_single().bc_interface.sync_wallet(wallet) + + #Trigger PING LAG sending artificially + joinmarket.irc.PING_INTERVAL = 3 + + mcs = [IRCMessageChannel(c) for c in get_irc_mchannels()] + mcc = MessageChannelCollection(mcs) + #add all utxo in mixdepth 0 to 'used' list of commitments, + utxos = wallet.get_utxos_by_mixdepth()[0] + for u, addrval in utxos.iteritems(): + priv = wallet.get_key_from_addr(addrval['address']) + podle = btc.PoDLE(u, priv) + for i in range(tries): + #loop because we want to use up all retries of this utxo + commitment = podle.generate_podle(i)['commit'] + btc.update_commitments(commitment=commitment) + + #create a new utxo, notionally from an external source; to make life a little + #easier we'll pay to another mixdepth, but this is OK because + #taker does not source from here currently, only from the utxos chosen + #for the transaction, not the whole wallet. So we can treat it as if + #external (don't access its privkey). + utxos = wallet.get_utxos_by_mixdepth()[1] + ecs = {} + for u, addrval in utxos.iteritems(): + priv = wallet.get_key_from_addr(addrval['address']) + ecs[u] = {} + ecs[u]['reveal']={} + for j in range(tries): + P, P2, s, e, commit = generate_single_podle_sig( + binascii.unhexlify(priv), j) + if 'P' not in ecs[u]: + ecs[u]['P'] = P + ecs[u]['reveal'][j] = {'P2':P2, 's':s, 'e':e} + btc.update_commitments(external_to_add=ecs) + #Now the conditions described above hold. We do a normal single + #sendpayment. + taker = sendpayment.SendPayment(mcc, wallet, destaddr, amount, 2, + 5000, 3, 0, True, + weighted_order_choose) + try: + log.debug('starting message channels') + mcc.run() + finally: + if any(yigen_procs): + for ygp in yigen_procs: + #NB *GENTLE* shutdown is essential for + #test coverage reporting! + ygp.send_signal(signal.SIGINT) + ygp.wait() + #wait for block generation + time.sleep(5) + received = jm_single().bc_interface.get_received_by_addr( + [destaddr], None)['data'][0]['balance'] + assert received == amount, "sendpayment failed - coins not arrived, " +\ + "received: " + str(received) + #Cleanup - remove the external commitments added + btc.update_commitments(external_to_remove=ecs) + +@pytest.mark.parametrize( + "consume_tx, age_required, cmt_age", + [ + (True, 9, 5), + (True, 9, 12), + ]) +def test_tx_commitments_used(setup_podle, consume_tx, age_required, cmt_age): + tries = jm_single().config.getint("POLICY","taker_utxo_retries") + #remember and reset at the end + taker_utxo_age = jm_single().config.getint("POLICY", "taker_utxo_age") + jm_single().config.set("POLICY", "taker_utxo_age", str(age_required)) + #Don't want to wait too long, but must account for possible + #throttling with !auth + jm_single().maker_timeout_sec = 12 + amount = 0 + wallets = make_wallets(3, + wallet_structures=[[1,2,1,0,0],[1,2,0,0,0],[2,2,1,0,0]], + mean_amt=1) + #the sendpayment bot uses the last wallet in the list + wallet = wallets[2]['wallet'] + + #make_wallets calls grab_coins which mines 1 block per individual payout, + #so the age of the coins depends on where they are in that list. The sendpayment + #is the last wallet in the list, and we choose the non-tx utxos which are in + #mixdepth 1 and 2 (2 and 1 utxos in each respectively). We filter for those + #that have sufficient age, so to get 1 which is old enough, it will be the oldest, + #which will have an age of 2 + 1 (the first utxo spent to that wallet). + #So if we need an age of 6, we need to mine 3 more blocks. + blocks_reqd = cmt_age - 3 + jm_single().bc_interface.tick_forward_chain(blocks_reqd) + yigen_procs = [] + for i in range(2): + ygp = local_command([python_cmd, yg_cmd,\ + str(wallets[i]['seed'])], bg=True) + time.sleep(2) #give it a chance + yigen_procs.append(ygp) + + time.sleep(5) + destaddr = btc.privkey_to_address( + binascii.hexlify(os.urandom(32)), + magicbyte=get_p2pk_vbyte()) + addr_valid, errormsg = validate_address(destaddr) + assert addr_valid, "Invalid destination address: " + destaddr + \ + ", error message: " + errormsg + + log.debug('starting sendpayment') + + jm_single().bc_interface.sync_wallet(wallet) + log.debug("Here is the whole wallet: \n" + str(wallet.unspent)) + #Trigger PING LAG sending artificially + joinmarket.irc.PING_INTERVAL = 3 + + mcs = [IRCMessageChannel(c) for c in get_irc_mchannels()] + mcc = MessageChannelCollection(mcs) + if consume_tx: + #add all utxo in mixdepth 0 to 'used' list of commitments, + utxos = wallet.get_utxos_by_mixdepth()[0] + for u, addrval in utxos.iteritems(): + priv = wallet.get_key_from_addr(addrval['address']) + podle = btc.PoDLE(u, priv) + for i in range(tries): + #loop because we want to use up all retries of this utxo + commitment = podle.generate_podle(i)['commit'] + btc.update_commitments(commitment=commitment) + + #Now test a sendpayment from mixdepth 0 with all the depth 0 utxos + #used up, so that the other utxos in the wallet get used. + taker = sendpayment.SendPayment(mcc, wallet, destaddr, amount, 2, + 5000, 3, 0, True, + weighted_order_choose) + try: + log.debug('starting message channels') + mcc.run() + finally: + if any(yigen_procs): + for ygp in yigen_procs: + #NB *GENTLE* shutdown is essential for + #test coverage reporting! + ygp.send_signal(signal.SIGINT) + ygp.wait() + #wait for block generation + time.sleep(5) + received = jm_single().bc_interface.get_received_by_addr( + [destaddr], None)['data'][0]['balance'] + jm_single().config.set("POLICY", "taker_utxo_age", str(taker_utxo_age)) + if cmt_age < age_required: + assert received == 0, "Coins arrived but shouldn't" + else: + assert received != 0, "sendpayment failed - coins not arrived, " +\ + "received: " + str(received) + +def test_external_commitments(setup_podle): + """Add this generated commitment to the external list + {txid:N:{'P':pubkey, 'reveal':{1:{'P2':P2,'s':s,'e':e}, 2:{..},..}}} + Note we do this *after* the sendpayment test so that the external + commitments will not erroneously used (they are fake). + """ + ecs = {} + tries = jm_single().config.getint("POLICY","taker_utxo_retries") + for i in range(10): + priv = os.urandom(32) + dummy_utxo = btc.sha256(priv)+":2" + ecs[dummy_utxo] = {} + ecs[dummy_utxo]['reveal']={} + for j in range(tries): + P, P2, s, e, commit = generate_single_podle_sig(priv, j) + if 'P' not in ecs[dummy_utxo]: + ecs[dummy_utxo]['P']=P + ecs[dummy_utxo]['reveal'][j] = {'P2':P2, 's':s, 'e':e} + btc.add_external_commitments(ecs) + used, external = btc.get_podle_commitments() + for u in external: + assert external[u]['P'] == ecs[u]['P'] + for i in range(tries): + for x in ['P2', 's', 'e']: + assert external[u]['reveal'][str(i)][x] == ecs[u]['reveal'][i][x] + +@pytest.fixture(scope="module") +def setup_podle(request): + load_program_config() + prev_commits = False + #back up any existing commitments + pcf = btc.get_commitment_file() + log.debug("Podle file: " + pcf) + if os.path.exists(pcf): + os.rename(pcf, pcf + ".bak") + prev_commits = True + def teardown(): + if prev_commits: + os.rename(pcf + ".bak", pcf) + else: + os.remove(pcf) + request.addfinalizer(teardown) diff --git a/test/test_regtest.py b/test/test_regtest.py index 64f1aaf5..e85a4b0f 100644 --- a/test/test_regtest.py +++ b/test/test_regtest.py @@ -6,11 +6,12 @@ import signal from commontest import local_command, make_wallets import os +import shutil import pytest import time from joinmarket import Taker, load_program_config, IRCMessageChannel -from joinmarket import validate_address, jm_single -from joinmarket import random_nick, get_p2pk_vbyte +from joinmarket import validate_address, jm_single, get_irc_mchannels +from joinmarket import random_nick, get_p2pk_vbyte, MessageChannelCollection from joinmarket import get_log, choose_sweep_orders, choose_orders, \ pick_order, cheapest_order_choose, weighted_order_choose, debug_dump_object import joinmarket.irc @@ -21,22 +22,30 @@ #for running bots as subprocesses python_cmd = 'python2' yg_cmd = 'yield-generator-basic.py' -#yg_cmd = 'yield-generator-mixdepth.py' -#yg_cmd = 'yield-generator-deluxe.py' - +#yg_cmd = 'yg-pe.py' @pytest.mark.parametrize( - "num_ygs, wallet_structures, mean_amt, mixdepth, sending_amt", + "num_ygs, wallet_structures, mean_amt, mixdepth, sending_amt, ygcfs, fails, donate", [ - # basic 1sp 2yg - (2, [[1, 0, 0, 0, 0]] * 3, 10, 0, 100000000), - # 1sp 4yg, 2 mixdepths - (4, [[1, 2, 0, 0, 0]] * 5, 4, 1, 1234500), - # 1sp 8yg, 4 mixdepths, sweep from depth 0 - (8, [[1, 3, 0, 0, 0]] * 9, 4, 0, 0), + #Some tests are commented out to keep build test time reasonable. + # basic 1sp 2yg. + #(4, [[1, 0, 0, 0, 0]] * 5, 10, 0, 100000000, None, None, 0.5), + (4, [[1, 0, 0, 0, 0]] * 5, 10, 0, 100000000, None, None, None), + #Testing different message channel collections. (Needs manual config at + #the moment - create different config files for each yg). + #(4, [[1, 0, 0, 0, 0]] * 5, 10, 0, 100000000, ["j2.cfg", "j3.cfg", + # "j4.cfg", "j5.cfg"], None), + # 1sp 3yg, 2 mixdepths - testing different failure times to + #see if recovery works. + #(5, [[1, 2, 0, 0, 0]] * 6, 4, 1, 1234500, None, None), + (4, [[1, 2, 0, 0, 0]] * 5, 4, 1, 1234500, None, ('break',0,6), None), + #(5, [[1, 2, 0, 0, 0]] * 6, 4, 1, 1234500, None, ('shutdown',0,12)), + #(5, [[1, 2, 0, 0, 0]] * 6, 4, 1, 1234500, None, ('break',1, 6)), + # 1sp 6yg, 4 mixdepths, sweep from depth 0 (test large number of makers) + (8, [[1, 3, 0, 0, 0]] * 9, 4, 0, 0, None, None, None), ]) def test_sendpayment(setup_regtest, num_ygs, wallet_structures, mean_amt, - mixdepth, sending_amt): + mixdepth, sending_amt, ygcfs, fails, donate): """Test of sendpayment code, with yield generators in background. """ log = get_log() @@ -52,33 +61,38 @@ def test_sendpayment(setup_regtest, num_ygs, wallet_structures, mean_amt, wallet = wallets[makercount]['wallet'] yigen_procs = [] + if ygcfs: + assert makercount == len(ygcfs) for i in range(makercount): + if ygcfs: + #back up default config, overwrite before start + os.rename("joinmarket.cfg", "joinmarket.cfg.bak") + shutil.copy2(ygcfs[i], "joinmarket.cfg") ygp = local_command([python_cmd, yg_cmd,\ str(wallets[i]['seed'])], bg=True) time.sleep(2) #give it a chance yigen_procs.append(ygp) + if ygcfs: + #Note: in case of using multiple configs, + #the starting config is what is used by sendpayment + os.rename("joinmarket.cfg.bak", "joinmarket.cfg") #A significant delay is needed to wait for the yield generators to sync time.sleep(20) - if btc.secp_present: - destaddr = btc.privkey_to_address( - os.urandom(32), - from_hex=False, - magicbyte=get_p2pk_vbyte()) + if donate: + destaddr = None else: destaddr = btc.privkey_to_address( os.urandom(32), + from_hex=False, magicbyte=get_p2pk_vbyte()) - - addr_valid, errormsg = validate_address(destaddr) - assert addr_valid, "Invalid destination address: " + destaddr + \ + addr_valid, errormsg = validate_address(destaddr) + assert addr_valid, "Invalid destination address: " + destaddr + \ ", error message: " + errormsg #TODO paramatetrize this as a test variable chooseOrdersFunc = weighted_order_choose - jm_single().nickname = random_nick() - log.debug('starting sendpayment') jm_single().bc_interface.sync_wallet(wallet) @@ -86,15 +100,17 @@ def test_sendpayment(setup_regtest, num_ygs, wallet_structures, mean_amt, #Trigger PING LAG sending artificially joinmarket.irc.PING_INTERVAL = 3 - irc = IRCMessageChannel(jm_single().nickname) + mcs = [IRCMessageChannel(c) for c in get_irc_mchannels()] + mcc = MessageChannelCollection(mcs) #hack fix for #356 if multiple orders per counterparty - if amount==0: makercount=2 - taker = sendpayment.SendPayment(irc, wallet, destaddr, amount, makercount, + #removed for now. + #if amount==0: makercount=2 + taker = sendpayment.SendPayment(mcc, wallet, destaddr, amount, makercount-2, txfee, waittime, mixdepth, answeryes, chooseOrdersFunc) try: - log.debug('starting irc') - irc.run() + log.debug('starting message channels') + mcc.run(failures=fails) finally: if any(yigen_procs): for ygp in yigen_procs: @@ -104,14 +120,15 @@ def test_sendpayment(setup_regtest, num_ygs, wallet_structures, mean_amt, ygp.wait() #wait for block generation time.sleep(5) - received = jm_single().bc_interface.get_received_by_addr( - [destaddr], None)['data'][0]['balance'] - if amount != 0: - assert received == amount, "sendpayment failed - coins not arrived, " +\ - "received: " + str(received) - #TODO: how to check success for sweep case? - else: - assert received != 0 + if not donate: + received = jm_single().bc_interface.get_received_by_addr( + [destaddr], None)['data'][0]['balance'] + if amount != 0: + assert received == amount, "sendpayment failed - coins not arrived, " +\ + "received: " + str(received) + #TODO: how to check success for sweep case? + else: + assert received != 0 @pytest.fixture(scope="module") diff --git a/test/test_tumbler.py b/test/test_tumbler.py index 7b2a490f..c6f11752 100644 --- a/test/test_tumbler.py +++ b/test/test_tumbler.py @@ -9,8 +9,8 @@ import pytest import time from joinmarket import Taker, load_program_config, IRCMessageChannel -from joinmarket import validate_address, jm_single -from joinmarket import random_nick, get_p2pk_vbyte +from joinmarket import validate_address, jm_single, MessageChannelCollection +from joinmarket import random_nick, get_p2pk_vbyte, get_irc_mchannels from joinmarket import get_log, choose_sweep_orders, choose_orders, \ pick_order, cheapest_order_choose, weighted_order_choose, debug_dump_object import json @@ -108,15 +108,10 @@ def test_tumbler(setup_tumbler, num_ygs, wallet_structures, mean_amt, sdev_amt, time.sleep(20) destaddrs = [] for i in range(3): - if btc.secp_present: - destaddr = btc.privkey_to_address( - os.urandom(32), - from_hex=False, - magicbyte=get_p2pk_vbyte()) - else: - destaddr = btc.privkey_to_address( - os.urandom(32), - magicbyte=get_p2pk_vbyte()) + destaddr = btc.privkey_to_address( + os.urandom(32), + from_hex=False, + magicbyte=get_p2pk_vbyte()) addr_valid, errormsg = validate_address(destaddr) assert addr_valid, "Invalid destination address: " + destaddr + \ ", error message: " + errormsg @@ -156,11 +151,12 @@ def test_tumbler(setup_tumbler, num_ygs, wallet_structures, mean_amt, sdev_amt, jm_single().bc_interface.sync_wallet(wallet) jm_single().bc_interface.pushtx_failure_prob = 0.4 - irc = IRCMessageChannel(jm_single().nickname) - tumbler_bot = tumbler.Tumbler(irc, wallet, tx_list, options) + mcs = [IRCMessageChannel(c, jm_single().nickname) for c in get_irc_mchannels()] + mcc = MessageChannelCollection(mcs) + tumbler_bot = tumbler.Tumbler(mcc, wallet, tx_list, options) try: - log.debug('starting irc') - irc.run() + log.debug('starting message channels') + mcc.run() except: log.debug('CRASHING, DUMPING EVERYTHING') debug_dump_object(wallet, ['addr_cache', 'keys', 'wallet_name', 'seed']) diff --git a/test/test_wallets.py b/test/test_wallets.py index 47be17c7..0045bedb 100644 --- a/test/test_wallets.py +++ b/test/test_wallets.py @@ -10,6 +10,7 @@ import random import subprocess import unittest +from decimal import Decimal from commontest import local_command, interact, make_wallets, make_sign_and_push import bitcoin as btc @@ -35,6 +36,42 @@ def do_tx(wallet, amount): time.sleep(2) #blocks jm_single().bc_interface.sync_unspent(wallet) +""" +@pytest.mark.parametrize( + "num_txs, gap_count, gap_limit, wallet_structure, amount, wallet_file, password", + [ + (3, 450, 461, [11,3,4,5,6], 150000000, 'test_import_wallet.json', 'import-pwd' + ), + ]) +def test_wallet_gap_sync(setup_wallets, num_txs, gap_count, gap_limit, + wallet_structure, amount, wallet_file, password): + #Starting with a nonexistent index_cache, try syncing with a large + #gap limit + setup_import(mainnet=False) + wallet = make_wallets(1,[wallet_structure], + fixed_seeds=[wallet_file], + test_wallet=True, passwords=[password])[0]['wallet'] + wallet.gaplimit = gap_limit + #Artificially insert coins at position (0, wallet_structures[0] + gap_count) + dest = wallet.get_addr(0, 0, wallet_structure[0] + gap_count) + btcamt = amount/(1e8) + jm_single().bc_interface.grab_coins(dest, amt=float(Decimal(btcamt).quantize(Decimal(10)**-8))) + time.sleep(2) + sync_count = 0 + jm_single().bc_interface.wallet_synced = False + while not jm_single().bc_interface.wallet_synced: + wallet.index = [] + for i in range(5): + wallet.index.append([0, 0]) + jm_single().bc_interface.sync_wallet(wallet) + sync_count += 1 + #avoid infinite loop + assert sync_count < 10 + log.debug("Tried " + str(sync_count) + " times") + + assert jm_single().bc_interface.wallet_synced +""" + @pytest.mark.parametrize( "num_txs, fake_count, wallet_structure, amount, wallet_file, password", [ diff --git a/test/ygrunner.py b/test/ygrunner.py new file mode 100644 index 00000000..c57dc4ab --- /dev/null +++ b/test/ygrunner.py @@ -0,0 +1,73 @@ +#! /usr/bin/env python +from __future__ import absolute_import +'''Creates wallets and yield generators in regtest. + Provides seed for joinmarket-qt test. + This should be run via pytest, even though + it's NOT part of the test-suite, because that + makes it much easier to handle start up and + shut down of the environment. + Run it like: + PYTHONPATH=.:$PYTHONPATH py.test \ + --btcroot=/path/to/bitcoin/bin/ \ + --btcpwd=123456abcdef --btcconf=/blah/bitcoin.conf \ + --nirc=2 -s test/ygrunner.py + ''' + +import subprocess +import signal +from commontest import local_command, make_wallets +import os +import pytest +import sys +import time +from joinmarket import load_program_config, jm_single + +#for running bots as subprocesses +python_cmd = 'python2' +yg_cmd = 'yield-generator-basic.py' +#yg_cmd = 'yield-generator-mixdepth.py' +#yg_cmd = 'yield-generator-deluxe.py' + +@pytest.mark.parametrize( + "num_ygs, wallet_structures, mean_amt", + [ + # 1sp 3yg, 2 mixdepths, sweep from depth1 + (4, [[1, 3, 0, 0, 0]] * 5, 2), + ]) +def test_start_ygs(setup_ygrunner, num_ygs, wallet_structures, mean_amt): + """Set up some wallets, for the ygs and 1 sp. + Then start the ygs in background and publish + the seed of the sp wallet for easy import into -qt + """ + wallets = make_wallets(num_ygs + 1, + wallet_structures=wallet_structures, + mean_amt=mean_amt) + #the sendpayment bot uses the last wallet in the list + wallet = wallets[num_ygs]['wallet'] + print "Seed : " + wallets[num_ygs]['seed'] + #useful to see the utxos on screen sometimes + jm_single().bc_interface.sync_wallet(wallet) + print wallet.unspent + + yigen_procs = [] + for i in range(num_ygs): + ygp = local_command([python_cmd, yg_cmd,\ + str(wallets[i]['seed'])], bg=True) + time.sleep(2) #give it a chance + yigen_procs.append(ygp) + try: + while True: + time.sleep(20) + print 'waiting' + finally: + if any(yigen_procs): + for ygp in yigen_procs: + #NB *GENTLE* shutdown is essential for + #test coverage reporting! + ygp.send_signal(signal.SIGINT) + ygp.wait() + +@pytest.fixture(scope="module") +def setup_ygrunner(): + load_program_config() + \ No newline at end of file diff --git a/tumbler.py b/tumbler.py index 90f06831..43e2e0af 100644 --- a/tumbler.py +++ b/tumbler.py @@ -12,12 +12,12 @@ from pprint import pprint from joinmarket import jm_single, Taker, load_program_config, \ - IRCMessageChannel + IRCMessageChannel, MessageChannelCollection from joinmarket import validate_address from joinmarket import random_nick from joinmarket import get_log, rand_norm_array, rand_pow_array, \ rand_exp_array, choose_orders, weighted_order_choose, choose_sweep_orders, \ - debug_dump_object + debug_dump_object, get_irc_mchannels from joinmarket import Wallet from joinmarket.wallet import estimate_tx_fee @@ -345,14 +345,14 @@ def run(self): sqlorders = self.taker.db.execute( 'SELECT cjfee, ordertype FROM orderbook;').fetchall() - orders = [o['cjfee'] for o in sqlorders if o['ordertype'] == 'relorder'] + orders = [o['cjfee'] for o in sqlorders if o['ordertype'] == 'reloffer'] orders = sorted(orders) if len(orders) == 0: log.debug('There are no orders at all in the orderbook! ' 'Is the bot connecting to the right server?') return relorder_fee = float(orders[0]) - log.debug('relorder fee = ' + str(relorder_fee)) + log.debug('reloffer fee = ' + str(relorder_fee)) maker_count = sum([tx['makercount'] for tx in self.taker.tx_list]) log.debug('uses ' + str(maker_count) + ' makers, at ' + str( relorder_fee * 100) + '% per maker, estimated total cost ' + str( @@ -633,15 +633,14 @@ def main(): wallet = Wallet(wallet_file, max_mix_depth=options['mixdepthsrc'] + options['mixdepthcount']) jm_single().bc_interface.sync_wallet(wallet) - - jm_single().nickname = random_nick() - + jm_single().wait_for_commitments = 1 log.debug('starting tumbler') - irc = IRCMessageChannel(jm_single().nickname) - tumbler = Tumbler(irc, wallet, tx_list, options) + mcs = [IRCMessageChannel(c) for c in get_irc_mchannels()] + mcc = MessageChannelCollection(mcs) + tumbler = Tumbler(mcc, wallet, tx_list, options) try: - log.debug('connecting to irc') - irc.run() + log.debug('connecting to message channels') + mcc.run() except: log.debug('CRASHING, DUMPING EVERYTHING') debug_dump_object(wallet, ['addr_cache', 'keys', 'seed']) diff --git a/wallet-tool.py b/wallet-tool.py index 7b6a6f18..03d50c68 100644 --- a/wallet-tool.py +++ b/wallet-tool.py @@ -20,7 +20,11 @@ 'balances. (displayall) Shows ALL addresses and balances. ' '(summary) Shows a summary of mixing depth balances. (generate) ' 'Generates a new wallet. (recover) Recovers a wallet from the 12 ' - 'word recovery seed. (showseed) Shows the wallet recovery seed ' + 'word recovery seed. (showutxos) Shows all utxos in the wallet, ' + 'including the corresponding private keys if -p is chosen; the ' + 'data is also written to a file "walletname.json.utxos" if the ' + 'option -u is chosen (so be careful about private keys). ' + '(showseed) Shows the wallet recovery seed ' 'and hex seed. (importprivkey) Adds privkeys to this wallet, ' 'privkeys are spaces or commas separated. (listwallets) Lists ' 'all wallets with creator and timestamp. (history) Show all ' @@ -70,7 +74,7 @@ noseed_methods = ['generate', 'recover', 'listwallets'] methods = ['display', 'displayall', 'summary', 'showseed', 'importprivkey', - 'history'] + 'history', 'showutxos'] methods.extend(noseed_methods) noscan_methods = ['showseed', 'importprivkey'] @@ -102,6 +106,20 @@ jm_single().config.set('POLICY','listunspent_args', '[0]') jm_single().bc_interface.sync_wallet(wallet) +if method == 'showutxos': + unsp = {} + if options.showprivkey: + for u, av in wallet.unspent.iteritems(): + addr = av['address'] + key = wallet.get_key_from_addr(addr) + wifkey = btc.wif_compressed_privkey(key, vbyte=get_p2pk_vbyte()) + unsp[u] = {'address': av['address'], + 'value': av['value'], 'privkey': wifkey} + else: + unsp = wallet.unspent + print(json.dumps(unsp, indent=4)) + sys.exit(0) + if method == 'display' or method == 'displayall' or method == 'summary': def cus_print(s): @@ -129,12 +147,8 @@ def cus_print(s): balance_depth += balance used = ('used' if k < wallet.index[m][forchange] else ' new') if options.showprivkey: - if btc.secp_present: - privkey = btc.wif_compressed_privkey( + privkey = btc.wif_compressed_privkey( wallet.get_key(m, forchange, k), get_p2pk_vbyte()) - else: - privkey = btc.encode_privkey(wallet.get_key(m, - forchange, k), 'wif_compressed', get_p2pk_vbyte()) else: privkey = '' if (method == 'displayall' or balance > 0 or @@ -153,12 +167,8 @@ def cus_print(s): used = (' used' if balance > 0.0 else 'empty') balance_depth += balance if options.showprivkey: - if btc.secp_present: - wip_privkey = btc.wif_compressed_privkey( + wip_privkey = btc.wif_compressed_privkey( privkey, get_p2pk_vbyte()) - else: - wip_privkey = btc.encode_privkey(privkey, - 'wif_compressed', get_p2pk_vbyte()) else: wip_privkey = '' cus_print(' ' * 13 + '%-35s%s %.8f btc %s' % ( @@ -222,29 +232,8 @@ def cus_print(s): for privkey in privkeys: # TODO is there any point in only accepting wif format? check what # other wallets do - if not btc.secp_present: - privkey_format = btc.get_privkey_format(privkey) - if privkey_format not in ['wif', 'wif_compressed']: - print('ERROR: privkey not in wallet import format') - print(privkey, 'skipped') - continue - if privkey_format == 'wif': - # TODO if they actually use an unc privkey, make sure the unc - # address is used - - # r = raw_input('WARNING: Using uncompressed private key, the vast ' + - # 'majority of JoinMarket transactions use compressed keys\n' + - # 'being so unusual is bad for privacy. Continue? (y/n):') - # if r != 'y': - # sys.exit(0) - print('Uncompressed privkeys not supported (yet)') - print(privkey, 'skipped') - continue - if btc.secp_present: - privkey_bin = btc.from_wif_privkey(privkey, + privkey_bin = btc.from_wif_privkey(privkey, vbyte=get_p2pk_vbyte()).decode('hex')[:-1] - else: - privkey_bin = btc.encode_privkey(privkey, 'hex').decode('hex') encrypted_privkey = encryptData(wallet.password_key, privkey_bin) if 'imported_keys' not in wallet.walletdata: wallet.walletdata['imported_keys'] = [] diff --git a/yg-pe.py b/yg-pe.py new file mode 100644 index 00000000..ed676fbf --- /dev/null +++ b/yg-pe.py @@ -0,0 +1,177 @@ +#! /usr/bin/env python +from __future__ import print_function + +import datetime +import os +import time + +from joinmarket import jm_single, get_network, load_program_config +from joinmarket import get_log, calc_cj_fee, debug_dump_object +from joinmarket import Wallet +from joinmarket import get_irc_mchannels +from joinmarket import YieldGenerator, ygmain + +txfee = 1000 +cjfee_a = 200 +cjfee_r = '0.002' +ordertype = 'reloffer' +nickserv_password = '' +minsize = 100000 +mix_levels = 5 + + +log = get_log() + +# is a maker for the purposes of generating a yield from held +# bitcoins while maximising the difficulty of spying on activity; +# this is primarily attempted by avoiding reannouncemnt of orders +# after transactions whereever that is possible. +class YieldGeneratorPrivEnhance(YieldGenerator): + + + def __init__(self, msgchan, wallet, offerconfig): + self.txfee, self.cjfee_a, self.cjfee_r, self.ordertype, self.minsize, \ + self.mix_levels = offerconfig + super(YieldGeneratorPrivEnhance,self).__init__(msgchan, wallet) + + def create_my_orders(self): + mix_balance = self.wallet.get_balance_by_mixdepth() + #We publish ONLY the maximum amount and use minsize for lower bound; + #leave it to oid_to_order to figure out the right depth to use. + f = '0' + if ordertype == 'reloffer': + f = self.cjfee_r + #minimum size bumped if necessary such that you always profit + #least 50% of the miner fee + self.minsize = int(1.5 * self.txfee / float(self.cjfee_r)) + elif ordertype == 'absoffer': + f = str(self.txfee + self.cjfee_a) + mix_balance = dict([(m, b) for m, b in mix_balance.iteritems() + if b > self.minsize]) + if len(mix_balance) == 0: + log.debug('do not have any coins left') + return [] + max_mix = max(mix_balance, key=mix_balance.get) + order = {'oid': 0, + 'ordertype': self.ordertype, + 'minsize': self.minsize, + 'maxsize': mix_balance[max_mix] - max( + jm_single().DUST_THRESHOLD, self.txfee), + 'txfee': self.txfee, + 'cjfee': f} + + # sanity check + assert order['minsize'] >= 0 + assert order['maxsize'] > 0 + assert order['minsize'] <= order['maxsize'] + + return [order] + + def oid_to_order(self, cjorder, oid, amount): + """The only change from *basic here (for now) is that + we choose outputs to avoid increasing the max_mixdepth + as much as possible, thus avoiding reannouncement as + much as possible. + """ + total_amount = amount + cjorder.txfee + mix_balance = self.wallet.get_balance_by_mixdepth() + max_mix = max(mix_balance, key=mix_balance.get) + min_mix = min(mix_balance, key=mix_balance.get) + + filtered_mix_balance = [m + for m in mix_balance.iteritems() + if m[1] >= total_amount] + if not filtered_mix_balance: + return None, None, None + + log.debug('mix depths that have enough = ' + str(filtered_mix_balance)) + + #Avoid the max mixdepth wherever possible, to avoid changing the + #offer. Algo: + #"mixdepth" is the mixdepth we are spending FROM, so it is also + #the destination of change. + #"cjoutdepth" is the mixdepth we are sending coinjoin out to. + # + #Find a mixdepth, in the set that have enough, which is + #not the maximum, and choose any from that set as "mixdepth". + #If not possible, it means only the max_mix depth has enough, + #so must choose "mixdepth" to be that. + #To find the cjoutdepth: ensure that max != min, if so it means + #we had only one depth; in that case, just set "cjoutdepth" + #to the next mixdepth. Otherwise, we set "cjoutdepth" to the minimum. + + nonmax_mix_balance = [m for m in filtered_mix_balance if m[0] != max_mix] + if not nonmax_mix_balance: + log.debug("Could not spend from a mixdepth which is not max") + mixdepth = max_mix + else: + mixdepth = nonmax_mix_balance[0][0] + log.debug('filling offer, mixdepth=' + str(mixdepth)) + + # mixdepth is the chosen depth we'll be spending from + # min_mixdepth is the one we want to send our cjout TO, + # to minimize chance of it becoming the largest, and reannouncing offer. + if mixdepth == min_mix: + cjoutmix = (mixdepth + 1) % self.wallet.max_mix_depth + #don't send cjout to max + if cjoutmix == max_mix: + cjoutmix = (cjoutmix + 1) % self.wallet.max_mix_depth + else: + cjoutmix = min_mix + cj_addr = self.wallet.get_internal_addr(cjoutmix) + change_addr = self.wallet.get_internal_addr(mixdepth) + + utxos = self.wallet.select_utxos(mixdepth, total_amount) + my_total_in = sum([va['value'] for va in utxos.values()]) + real_cjfee = calc_cj_fee(cjorder.ordertype, cjorder.cjfee, amount) + change_value = my_total_in - amount - cjorder.txfee + real_cjfee + if change_value <= jm_single().DUST_THRESHOLD: + log.debug(('change value={} below dust threshold, ' + 'finding new utxos').format(change_value)) + try: + utxos = self.wallet.select_utxos( + mixdepth, total_amount + jm_single().DUST_THRESHOLD) + except Exception: + log.debug('dont have the required UTXOs to make a ' + 'output above the dust threshold, quitting') + return None, None, None + + return utxos, cj_addr, change_addr + + def on_tx_unconfirmed(self, cjorder, txid, removed_utxos): + self.tx_unconfirm_timestamp[cjorder.cj_addr] = int(time.time()) + # if the balance of the highest-balance mixing depth change then + # reannounce it + oldorder = self.orderlist[0] if len(self.orderlist) > 0 else None + neworders = self.create_my_orders() + if len(neworders) == 0: + return [0], [] # cancel old order + # oldorder may not exist when this is called from on_tx_confirmed + # (this happens when we just spent from the max mixdepth and so had + # to cancel the order). + if oldorder: + if oldorder['maxsize'] == neworders[0]['maxsize']: + return [], [] # change nothing + # announce new order, replacing the old order + return [], [neworders[0]] + + def on_tx_confirmed(self, cjorder, confirmations, txid): + if cjorder.cj_addr in self.tx_unconfirm_timestamp: + confirm_time = int(time.time()) - self.tx_unconfirm_timestamp[ + cjorder.cj_addr] + else: + confirm_time = 0 + timestamp = datetime.datetime.now().strftime("%Y/%m/%d %H:%M:%S") + self.log_statement([timestamp, cjorder.cj_amount, len( + cjorder.utxos), sum([av['value'] for av in cjorder.utxos.values( + )]), cjorder.real_cjfee, cjorder.real_cjfee - cjorder.txfee, round( + confirm_time / 60.0, 2), '']) + return self.on_tx_unconfirmed(cjorder, txid, None) + + +if __name__ == "__main__": + ygmain(YieldGeneratorPrivEnhance, txfee=txfee, + cjfee_a=cjfee_a, cjfee_r=cjfee_r, + ordertype=ordertype, nickserv_password=nickserv_password, + minsize=minsize, mix_levels=mix_levels) + print('done') diff --git a/yield-generator-basic.py b/yield-generator-basic.py index 6789c882..ae53f106 100644 --- a/yield-generator-basic.py +++ b/yield-generator-basic.py @@ -6,68 +6,32 @@ import time from optparse import OptionParser -from joinmarket import Maker, IRCMessageChannel -from joinmarket import BlockrInterface from joinmarket import jm_single, get_network, load_program_config -from joinmarket import random_nick from joinmarket import get_log, calc_cj_fee, debug_dump_object -from joinmarket import Wallet - -# data_dir = os.path.dirname(os.path.realpath(__file__)) -# sys.path.insert(0, os.path.join(data_dir, 'joinmarket')) - -# import blockchaininterface +from joinmarket import Wallet, YieldGenerator, ygmain +from joinmarket import get_irc_mchannels txfee = 1000 cjfee_a = 200 cjfee_r = '0.002' -ordertype = 'relorder' -jm_single().nickname = '' +ordertype = 'reloffer' nickserv_password = '' minsize = 100000 mix_levels = 5 - log = get_log() # is a maker for the purposes of generating a yield from held -# bitcoins without ruining privacy for the taker, the taker could easily check -# the history of the utxos this bot sends, so theres not much incentive -# to ruin the privacy for barely any more yield -# sell-side algorithm: -# add up the value of each utxo for each mixing depth, -# announce a relative-fee order of the highest balance -# spent from utxos that try to make the highest balance even higher -# so try to keep coins concentrated in one mixing depth -class YieldGenerator(Maker): - statement_file = os.path.join('logs', 'yigen-statement.csv') - - def __init__(self, msgchan, wallet): - Maker.__init__(self, msgchan, wallet) - self.msgchan.register_channel_callbacks(self.on_welcome, - self.on_set_topic, None, None, - self.on_nick_leave, None) - self.tx_unconfirm_timestamp = {} - - def log_statement(self, data): - if get_network() == 'testnet': - return - - data = [str(d) for d in data] - self.income_statement = open(self.statement_file, 'a') - self.income_statement.write(','.join(data) + '\n') - self.income_statement.close() - - def on_welcome(self): - Maker.on_welcome(self) - if not os.path.isfile(self.statement_file): - self.log_statement( - ['timestamp', 'cj amount/satoshi', 'my input count', - 'my input value/satoshi', 'cjfee/satoshi', 'earned/satoshi', - 'confirm time/min', 'notes']) +# bitcoins, offering from the maximum mixdepth and trying to offer +# the largest amount within the constraints of mixing depth isolation. +# It will often (but not always) reannounce orders after transactions, +# thus is somewhat suboptimal in giving more information to spies. +class YieldGeneratorBasic(YieldGenerator): - timestamp = datetime.datetime.now().strftime("%Y/%m/%d %H:%M:%S") - self.log_statement([timestamp, '', '', '', '', '', '', 'Connected']) + def __init__(self, msgchan, wallet, offerconfig): + self.txfee, self.cjfee_a, self.cjfee_r, self.ordertype, self.minsize, \ + self.mix_levels = offerconfig + super(YieldGeneratorBasic,self).__init__(msgchan, wallet) def create_my_orders(self): mix_balance = self.wallet.get_balance_by_mixdepth() @@ -78,9 +42,9 @@ def create_my_orders(self): # print mix_balance max_mix = max(mix_balance, key=mix_balance.get) f = '0' - if ordertype == 'relorder': + if ordertype == 'reloffer': f = cjfee_r - elif ordertype == 'absorder': + elif ordertype == 'absoffer': f = str(txfee + cjfee_a) order = {'oid': 0, 'ordertype': ordertype, @@ -161,91 +125,9 @@ def on_tx_confirmed(self, cjorder, confirmations, txid): confirm_time / 60.0, 2), '']) return self.on_tx_unconfirmed(cjorder, txid, None) - -def main(): - global txfee, cjfee_a, cjfee_r, ordertype, nickserv_password, minsize, mix_levels - import sys - - parser = OptionParser(usage='usage: %prog [options] [wallet file]') - parser.add_option('-o', '--ordertype', action='store', type='string', dest='ordertype', default=ordertype, - help='type of order; can be either relorder or absorder') - parser.add_option('-t', '--txfee', action='store', type='int', dest='txfee', default=txfee, - help='minimum miner fee in satoshis') - parser.add_option('-c', '--cjfee', action='store', type='string', dest='cjfee', default='', - help='requested coinjoin fee in satoshis or proportion') - parser.add_option('-n', '--nickname', action='store', type='string', dest='nickname', default=jm_single().nickname, - help='irc nickname') - parser.add_option('-p', '--password', action='store', type='string', dest='password', default=nickserv_password, - help='irc nickserv password') - parser.add_option('-s', '--minsize', action='store', type='int', dest='minsize', default=minsize, - help='minimum coinjoin size in satoshis') - parser.add_option('-m', '--mixlevels', action='store', type='int', dest='mixlevels', default=mix_levels, - help='number of mixdepths to use') - (options, args) = parser.parse_args() - if len(args) < 1: - parser.error('Needs a wallet') - sys.exit(0) - seed = args[0] - ordertype = options.ordertype - txfee = options.txfee - if ordertype == 'relorder': - if options.cjfee != '': - cjfee_r = options.cjfee - # minimum size is such that you always net profit at least 20% of the miner fee - minsize = max(int(1.2 * txfee / float(cjfee_r)), options.minsize) - elif ordertype == 'absorder': - if options.cjfee != '': - cjfee_a = int(options.cjfee) - minsize = options.minsize - else: - parser.error('You specified an incorrect order type which can be either relorder or absorder') - sys.exit(0) - if jm_single().nickname == options.nickname: - jm_single().nickname = random_nick() - else: - jm_single().nickname = options.nickname - nickserv_password = options.password - mix_levels = options.mixlevels - - load_program_config() - if isinstance(jm_single().bc_interface, BlockrInterface): - c = ('\nYou are running a yield generator by polling the blockr.io ' - 'website. This is quite bad for privacy. That site is owned by ' - 'coinbase.com Also your bot will run faster and more efficently, ' - 'you can be immediately notified of new bitcoin network ' - 'information so your money will be working for you as hard as ' - 'possibleLearn how to setup JoinMarket with Bitcoin Core: ' - 'https://github.com/chris-belcher/joinmarket/wiki/Running' - '-JoinMarket-with-Bitcoin-Core-full-node') - print(c) - ret = raw_input('\nContinue? (y/n):') - if ret[0] != 'y': - return - - wallet = Wallet(seed, max_mix_depth=mix_levels) - jm_single().bc_interface.sync_wallet(wallet) - - # nickname is set way above - # nickname - - log.debug('starting yield generator') - irc = IRCMessageChannel(jm_single().nickname, - realname='btcint=' + jm_single().config.get( - "BLOCKCHAIN", "blockchain_source"), - password=nickserv_password) - maker = YieldGenerator(irc, wallet) - try: - log.debug('connecting to irc') - irc.run() - except: - log.debug('CRASHING, DUMPING EVERYTHING') - debug_dump_object(wallet, ['addr_cache', 'keys', 'seed']) - debug_dump_object(maker) - debug_dump_object(irc) - import traceback - log.debug(traceback.format_exc()) - - if __name__ == "__main__": - main() + ygmain(YieldGeneratorBasic, txfee=txfee, cjfee_a=cjfee_a, + cjfee_r=cjfee_r, ordertype=ordertype, + nickserv_password=nickserv_password, + minsize=minsize, mix_levels=mix_levels) print('done') diff --git a/yield-generator-deluxe.py b/yield-generator-deluxe.py deleted file mode 100644 index 9067719f..00000000 --- a/yield-generator-deluxe.py +++ /dev/null @@ -1,605 +0,0 @@ -#! /usr/bin/env python -from __future__ import absolute_import, print_function - -import datetime -import os -import time -import binascii -import sys -import random -import decimal - -from joinmarket import Maker, IRCMessageChannel -from joinmarket import blockchaininterface, BlockrInterface -from joinmarket import jm_single, get_network, load_program_config -from joinmarket import random_nick -from joinmarket import get_log, calc_cj_fee, debug_dump_object -from joinmarket import Wallet - -#CONFIGURATION -mix_levels = 5 # Careful! Only change this if you setup your wallet as such. -nickname = random_nick() -nickserv_password = '' - -#Spread Types -# fibonacci- will gradually increase at the rate of the fibonacci sequence -# evenly- will be evenly spaced -# random- random amounts between the high and the low -# custom- use _custom to set it directly -# bymixdepth- (for offers), make offer amounts equal to mixdepths -# note, when using bymixdepth, set 'num_offers = mix_levels' - -# min and max offer sizes -offer_spread = 'fibonacci' # fibonacci, evenly, random, custom, bymixdepth -offer_low = None # satoshis. when None, min_output_size will be used -offer_high = None # satoshis. when None, size of largest mix depth will be used -#offer_low = random.randrange(21000000, 1e8) #random -#offer_high = random.randrange(150 * 1e8, 200 * 1e8) -custom_offers = [ - 1.01 * 1e8, 112345657, 1 * 1e8, 10 * 1e8, 100 * 1e8 -] # used when offer_spread is set to custom - -# percent fees for mix levels. -cjfee_spread = 'fibonacci' # fibonacci, evenly, random, custom -cjfee_low = random.uniform(0.0004, 0.0005) -cjfee_high = random.uniform(0.001, 0.01) -custom_cjfees = [ - 0.01, 0.0123, 0.013, 0.014, 0.015 -] # used when cjfee_spread is set to custom - -txfee_spread = 'fibonacci' # fibonacci, evenly, random, custom -txfee_low = 0 -txfee_high = 0 -#txfee_low = random.randrange(1, 100) -#txfee_high = random.randrange(3000, 5000) -custom_txfees = [ - 250, 500, 1000, 2000, 5000 -] # used when txfee_spread is set to custom - -# number of offers to autogenerate -num_offers = random.randrange(6, 9) # varied -#num_offers = 8 -#num_offers = mix_levels - -# only create change greater than this amount -min_output_size = random.randrange(15000, 300000) # varied -#min_output_size = random.randrange(5e6, 21e6) # varied -#min_output_size = 15000 -#min_output_size = jm_single().DUST_THRESHOLD # 546 satoshis - -# minimum profit you require for a transaction (exact absorders for dust exempt) -profit_req_per_transaction = 1 - -# You can override the above autogenerate options for maximum customization -override_offers = None # comment this line if using below -""" -override_offers = [ - {'ordertype': 'absorder', 'oid': 0, 'minsize': 0, 'maxsize': 100000000, 'cjfee': 0, 'txfee': 2000}, - {'ordertype': 'absorder', 'oid': 1, 'minsize': 0, 'maxsize': 1500000000, 'cjfee': 300000, 'txfee': 2000}, - {'ordertype': 'relorder', 'oid': 2, 'minsize': 15000, 'maxsize': 100000000, 'cjfee': 0.0001, 'txfee': 2000}, - {'ordertype': 'relorder', 'oid': 3, 'minsize': 15000, 'maxsize': 1000000000, 'cjfee': 0.0002, 'txfee': 2000}, - {'ordertype': 'relorder', 'oid': 4, 'minsize': 15000, 'maxsize': 2500000000, 'cjfee': 0.0003, 'txfee': 2000}, - ] -""" - -#END CONFIGURATION - -log = get_log() -if offer_low: - log.debug('offer_low = ' + str(offer_low) + " (" + str(offer_low / 1e8) + - " btc)") -if offer_high: - log.debug('offer_high = ' + str(offer_high) + " (" + str(offer_high / 1e8) + - " btc)") -log.debug('cjfee_low = ' + str(cjfee_low)) -log.debug('cjfee_high = ' + str(cjfee_high)) -log.debug('txfee_low = ' + str(txfee_low)) -log.debug('txfee_high = ' + str(txfee_high)) -log.debug('min_output_size = ' + str(min_output_size) + " (" + str( - min_output_size / 1e8) + " btc)") -profit_req_per_transaction = max(profit_req_per_transaction, -250) # safe guard -log.debug('profit_req_per_transaction = ' + str(profit_req_per_transaction)) - - -def fib(n): - a, b = 0, 1 - for i in range(n): - a, b = b, a + b - return a - - -def fib_seq(low, high, num, upper_bound=False): - x = [] - if upper_bound: - num += 1 - else: - x.append(low) - fib_sec = (high - low) / decimal.Decimal(fib(num)) - for y in range(2, (num + 1)): - x.append(low + (fib_sec * fib(y))) - return x - - -# range function for decimals -def drange(start, stop, step): - r = start - while r < stop: - yield r - r += step - yield stop - - -class YieldGenerator(Maker): - statement_file = os.path.join('logs', 'yigen-statement.csv') - - def __init__(self, msgchan, wallet): - Maker.__init__(self, msgchan, wallet) - self.msgchan.register_channel_callbacks(self.on_welcome, - self.on_set_topic, None, None, - self.on_nick_leave, None) - self.tx_unconfirm_timestamp = {} - - def log_statement(self, data): - if get_network() == 'testnet': - return - - data = [str(d) for d in data] - self.income_statement = open(self.statement_file, 'a') - self.income_statement.write(','.join(data) + '\n') - self.income_statement.close() - - def on_welcome(self): - Maker.on_welcome(self) - if not os.path.isfile(self.statement_file): - self.log_statement( - ['timestamp', 'cj amount/satoshi', 'my input count', - 'my input value/satoshi', 'cjfee/satoshi', 'earned/satoshi', - 'confirm time/min', 'notes']) - - timestamp = datetime.datetime.now().strftime("%Y/%m/%d %H:%M:%S") - self.log_statement([timestamp, '', '', '', '', '', '', 'Connected']) - - def create_my_orders(self): - mix_balance = self.wallet.get_balance_by_mixdepth() - log.debug('mix_balance = ' + str(mix_balance)) - log.debug('mix_btcance = ' + str([(x, y / 1e8) - for x, y in mix_balance.iteritems()])) - sorted_mix_balance = sorted( - list(mix_balance.iteritems()), - key=lambda a: a[1]) #sort by size - - largest_mixdepth_size = sorted_mix_balance[-1][1] - if largest_mixdepth_size <= min_output_size: - print("ALERT: not enough funds available in wallet") - return [] - - if override_offers: - log.debug('override_offers = \n' + '\n'.join([str( - o) for o in override_offers])) - # make sure custom offers dont create a negative net - for offer in override_offers: - if offer['ordertype'] == 'absorder': - profit = offer['cjfee'] - needed = 'make txfee be less then the cjfee' - elif offer['ordertype'] == 'relorder': - profit = calc_cj_fee(offer['ordertype'], offer['cjfee'], - offer['minsize']) - if float(offer['cjfee']) > 0: - needed = 'set minsize to ' + str(int(int(offer[ - 'txfee'] / float(offer['cjfee'])))) - if int(offer['txfee']) > profit: - print("ALERT: negative yield") - print('-> ' + str(offer)) - print(needed) - # if you really wanted to, you could comment out the next line. - sys.exit(0) - return override_offers - - offer_lowx = max(offer_low, min_output_size) - - if txfee_spread == 'custom': - maximum_conf_txfees = max(custom_txfees) - else: - maximum_conf_txfees = txfee_high - if offer_high: - offer_highx = min(offer_high, - largest_mixdepth_size - max(min_output_size,maximum_conf_txfees)) - else: - offer_highx = largest_mixdepth_size - max(min_output_size,maximum_conf_txfees) - # note, subtracting mix_output_size here to make minimum size change - # todo, make an offer for exactly the max size with no change - - # Offers - if offer_spread == 'fibonacci': - offer_levels = fib_seq(offer_lowx, - offer_highx, - num_offers, - upper_bound=True) - offer_levels = [int(round(x)) for x in offer_levels] - elif offer_spread == 'evenly': - first_upper_bound = (offer_highx - offer_lowx) / num_offers - offer_levels = list(range(first_upper_bound, offer_highx, ( - offer_highx - first_upper_bound) / (num_offers - 1))) - offer_levels = offer_levels[0:(num_offers - 1)] + [offer_highx] - elif offer_spread == 'random': - offer_levels = sorted([random.randrange(offer_lowx, offer_highx) - for n in range(num_offers - 1)] + - [random.randrange(offer_highx - ( - offer_highx / num_offers), offer_highx)]) - elif offer_spread == 'bymixdepth': - offer_levels = [] - for m in sorted_mix_balance: - if m[1] == 0: - continue - elif m[1] <= offer_lowx: - # todo, low mix balances get an absolute offer - continue - elif m[1] > offer_highx: - offer_levels += [offer_highx] - break - else: - offer_levels += [m[1]] - # note, offer_levels len can be less then num_offers here - elif offer_spread == 'custom': - assert len(custom_offers) == num_offers - offer_levels = [ - int((decimal.Decimal(str(x))).quantize(0)) - for x in sorted(custom_offers) - ] - if offer_levels[-1] > offer_highx: - log.debug( - 'ALERT: Your custom offers exceeds you max offer size.') - log.debug('offer = ' + str(offer_levels[-1]) + ' offer_highx = ' - + str(offer_highx)) - sys.exit(0) - else: - log.debug('invalid offer_spread = ' + str(offer_spread)) - sys.exit(0) - - # CJFees - cjfee_lowx = decimal.Decimal(str(cjfee_low)) / 100 - cjfee_highx = decimal.Decimal(str(cjfee_high)) / 100 - if cjfee_spread == 'fibonacci': - cjfee_levels = fib_seq(cjfee_lowx, cjfee_highx, num_offers) - cjfee_levels = ["%0.7f" % x for x in cjfee_levels] - elif cjfee_spread == 'evenly': - cjfee_levels = drange(cjfee_lowx, cjfee_highx, - (cjfee_highx - cjfee_lowx) / - (num_offers - 1)) # evenly spaced - cjfee_levels = ["%0.7f" % x for x in cjfee_levels] - elif cjfee_spread == 'random': - cjfee_levels = sorted( - ["%0.7f" % random.uniform( - float(cjfee_lowx), float(cjfee_highx)) - for n in range(num_offers)]) # randomly spaced - elif cjfee_spread == 'custom': - cjfee_levels = [str(decimal.Decimal(str(x)) / 100) - for x in custom_cjfees] - leftout = num_offers - len(cjfee_levels) - while leftout > 0: - log.debug('ALERT: cjfee_custom has too few items') - cjfee_levels.append(cjfee_levels[-1]) - leftout -= 1 - else: - log.debug('invalid cjfee_spread = ' + str(cjfee_spread)) - sys.exit(0) - - # TXFees - if txfee_spread == 'fibonacci': - txfee_levels = fib_seq(txfee_low, txfee_high, num_offers) - txfee_levels = [int(round(x)) for x in txfee_levels] - elif txfee_spread == 'evenly': - txfee_levels = list(range(txfee_low, txfee_high, ( - txfee_high - txfee_low) / (num_offers - 1))) - txfee_levels = txfee_levels[0:(num_offers - 1)] + [txfee_high] - elif txfee_spread == 'random': - txfee_levels = sorted([random.randrange(txfee_low, txfee_high) - for n in range(num_offers - 1)] + - [random.randrange(txfee_high - ( - txfee_high / num_offers), txfee_high)]) - elif txfee_spread == 'custom': - txfee_levels = [x for x in custom_txfees] - else: - log.debug('invalid txfee_spread = ' + str(txfee_spread)) - sys.exit(0) - - log.debug('offer_levels = ' + str(offer_levels)) - lower_bound_balances = [offer_lowx] + [x for x in offer_levels[:-1]] - if offer_spread == 'bymixdepth': - cjfee_levels = cjfee_levels[-len(offer_levels):] - txfee_levels = txfee_levels[-len(offer_levels):] - offer_ranges = zip(offer_levels, lower_bound_balances, cjfee_levels, - txfee_levels) - log.debug('offer_ranges = ' + str(offer_ranges)) - offers = [] - oid = 0 - - # create absorders for mixdepth dust - offer_levels = [] - for m in sorted_mix_balance: - if m[1] == 0: - continue - #elif False: # disabled - #elif m[1] <= 2e8: # absorder all mixdepths less then - elif m[1] <= offer_lowx: - offer = {'oid': oid, - 'ordertype': 'absorder', - 'minsize': m[1], - 'maxsize': m[1], - 'txfee': 0, - 'cjfee': 0} - #'txfee': txfee_low, - #'cjfee': min_revenue} - oid += 1 - offers.append(offer) - elif m[1] > offer_lowx: - break - - for upper, lower, cjfee, txfee in offer_ranges: - cjfee = float(cjfee) - if cjfee == 0: - min_needed = profit_req_per_transaction + txfee - elif cjfee > 0: - min_needed = int((profit_req_per_transaction + txfee + 1) / cjfee) - elif cjfee < 0: - sys.exit('negative fee not supported here') - if min_needed <= lower: - # create a regular relorder - offer = {'oid': oid, - 'ordertype': 'relorder', - 'minsize': lower, - 'maxsize': upper, - 'txfee': txfee, - 'cjfee': cjfee} - elif min_needed > lower and min_needed < upper: - # create two offers. An absolute for lower bound need, and relorder for the rest - offer = {'oid': oid, - 'ordertype': 'absorder', - 'minsize': lower, - 'maxsize': min_needed - 1, - 'txfee': txfee, - 'cjfee': profit_req_per_transaction + txfee} - oid += 1 - offers.append(offer) - offer = {'oid': oid, - 'ordertype': 'relorder', - 'minsize': min_needed, - 'maxsize': upper, - 'txfee': txfee, - 'cjfee': cjfee} - elif min_needed >= upper: - # just create an absolute offer - offer = {'oid': oid, - 'ordertype': 'absorder', - 'minsize': lower, - 'maxsize': upper, - 'txfee': txfee, - 'cjfee': profit_req_per_transaction + txfee} - # todo: combine neighboring absorders into a single one - oid += 1 - offers.append(offer) - - deluxe_offer_display = [] - header = 'oid'.rjust(5) - header += 'type'.rjust(7) - header += 'minsize btc'.rjust(15) - header += 'maxsize btc'.rjust(15) - header += 'min revenue satosh'.rjust(22) - header += 'max revenue satosh'.rjust(22) - deluxe_offer_display.append(header) - for o in offers: - line = str(o['oid']).rjust(5) - if o['ordertype'] == 'absorder': - line += 'abs'.rjust(7) - elif o['ordertype'] == 'relorder': - line += 'rel'.rjust(7) - line += str(o['minsize'] / 1e8).rjust(15) - line += str(o['maxsize'] / 1e8).rjust(15) - if o['ordertype'] == 'absorder': - line += str(o['cjfee']).rjust(22) - elif o['ordertype'] == 'relorder': - line += str(int(float(o['cjfee']) * int(o['minsize']))).rjust( - 22) - line += str(int(float(o['cjfee']) * int(o['maxsize']))).rjust( - 22) - deluxe_offer_display.append(line) - - log.debug('deluxe offer display = \n' + '\n'.join([str( - x) for x in deluxe_offer_display])) - - log.debug('generated offers = \n' + '\n'.join([str(o) for o in offers])) - - # sanity check - for offer in offers: - assert offer['minsize'] >= 0 - assert offer['maxsize'] > 0 - assert offer['minsize'] <= offer['maxsize'] - - return offers - - def oid_to_order(self, cjorder, oid, amount): - '''Coins rotate circularly from max mixdepth back to mixdepth 0''' - mix_balance = self.wallet.get_balance_by_mixdepth() - total_amount = amount + cjorder.txfee - log.debug('amount, txfee, total_amount = ' + str(amount) + str( - cjorder.txfee) + str(total_amount)) - - # look for exact amount available with no change - filtered_mix_balance = [m - for m in mix_balance.iteritems() - if m[1] == total_amount] - if filtered_mix_balance: - log.debug('mix depths that have the exact amount needed = ' + str( - filtered_mix_balance)) - else: - log.debug('no mix depths contain the exact amount needed.') - filtered_mix_balance = [m - for m in mix_balance.iteritems() - if m[1] >= total_amount] - log.debug('mix depths that have enough = ' + str( - filtered_mix_balance)) - filtered_mix_balance = [m - for m in mix_balance.iteritems() - if m[1] >= total_amount + min_output_size] - log.debug('mix depths that have enough with min_output_size, ' + - str(filtered_mix_balance)) - try: - len(filtered_mix_balance) > 0 - except Exception: - log.debug('No mix depths have enough funds to cover the ' + - 'amount, cjfee, and min_output_size.') - return None, None, None - - # prioritize by mixdepths sequencially - # keep coins moving towards last mixdepth, clumps once they get there - # makes sure coins sent to mixdepth 0 will get mixed to max mixdepth - filtered_mix_balance = sorted(filtered_mix_balance, key=lambda x: x[0]) - - # clumping. push all coins towards the largest mixdepth - # the largest amount of coins are available to join with (since joins always come from a single depth) - # the maker commands a higher fee for the larger amounts - # order ascending but circularly with largest last - # note, no need to consider max_offer_size here - #largest_mixdepth = sorted( - # filtered_mix_balance, - # key=lambda x: x[1],)[-1] # find largest amount - #smb = sorted(filtered_mix_balance, - # key=lambda x: x[0]) # seq of mixdepth num - #next_index = smb.index(largest_mixdepth) + 1 - #mmd = self.wallet.max_mix_depth - #filtered_mix_balance = smb[next_index % mmd:] + smb[:next_index % mmd] - - # use mix depth that has the closest amount of coins to what this transaction needs - # keeps coins moving through mix depths more quickly - # and its more likely to use txos of a similiar size to this transaction - # sort smallest to largest usable amount - #filtered_mix_balance = sorted(filtered_mix_balance, key=lambda x: x[1]) - - # use mix depth with the most coins, - # creates a more even distribution across mix depths - # and a more diverse txo selection in each depth - # sort largest to smallest amount - #filtered_mix_balance = sorted(filtered_mix_balance, key=lambda x: x[1], reverse=True) - - # use a random usable mixdepth. - # warning, could expose more txos to malicous taker requests - #filtered_mix_balance = random.choice(filtered_mix_balance) - - log.debug('sorted order of filtered_mix_balance = ' + str( - filtered_mix_balance)) - - mixdepth = filtered_mix_balance[0][0] - - log.debug('filling offer, mixdepth=' + str(mixdepth)) - - # mixdepth is the chosen depth we'll be spending from - cj_addr = self.wallet.get_internal_addr((mixdepth + 1) % - self.wallet.max_mix_depth) - change_addr = self.wallet.get_internal_addr(mixdepth) - - utxos = self.wallet.select_utxos(mixdepth, total_amount) - my_total_in = sum([va['value'] for va in utxos.values()]) - real_cjfee = calc_cj_fee(cjorder.ordertype, cjorder.cjfee, amount) - change_value = my_total_in - amount - cjorder.txfee + real_cjfee - if change_value <= min_output_size: - log.debug('change value=%d below dust threshold, finding new utxos' - % (change_value)) - try: - utxos = self.wallet.select_utxos(mixdepth, - total_amount + min_output_size) - except Exception: - log.debug( - 'dont have the required UTXOs to make a output above the dust threshold, quitting') - return None, None, None - - return utxos, cj_addr, change_addr - - def on_tx_unconfirmed(self, cjorder, txid, removed_utxos): - self.tx_unconfirm_timestamp[cjorder.cj_addr] = int(time.time()) - ''' - algorithm - find all the orders which have changed - ''' - - neworders = self.create_my_orders() - oldorders = self.orderlist - new_setdiff_old = [o for o in neworders if o not in oldorders] - old_setdiff_new = [o for o in oldorders if o not in neworders] - - log.debug('neworders = \n' + '\n'.join([str(o) for o in neworders])) - log.debug('oldorders = \n' + '\n'.join([str(o) for o in oldorders])) - log.debug('new_setdiff_old = \n' + '\n'.join([str( - o) for o in new_setdiff_old])) - log.debug('old_setdiff_new = \n' + '\n'.join([str( - o) for o in old_setdiff_new])) - - ann_orders = new_setdiff_old - ann_oids = [o['oid'] for o in ann_orders] - cancel_orders = [o['oid'] - for o in old_setdiff_new if o['oid'] not in ann_oids] - - log.debug('can_orders = \n' + '\n'.join([str(o) for o in cancel_orders - ])) - log.debug('ann_orders = \n' + '\n'.join([str(o) for o in ann_orders])) - - return (cancel_orders, ann_orders) - - def on_tx_confirmed(self, cjorder, confirmations, txid): - if cjorder.cj_addr in self.tx_unconfirm_timestamp: - confirm_time = int(time.time()) - self.tx_unconfirm_timestamp[ - cjorder.cj_addr] - else: - confirm_time = 0 - del self.tx_unconfirm_timestamp[cjorder.cj_addr] - timestamp = datetime.datetime.now().strftime("%Y/%m/%d %H:%M:%S") - self.log_statement([timestamp, cjorder.cj_amount, len( - cjorder.utxos), sum([av['value'] for av in cjorder.utxos.values( - )]), cjorder.real_cjfee, cjorder.real_cjfee - cjorder.txfee, round( - confirm_time / 60.0, 2), '']) - return self.on_tx_unconfirmed(cjorder, txid, None) - - -def main(): - load_program_config() - import sys - seed = sys.argv[1] - if isinstance(jm_single().bc_interface, - blockchaininterface.BlockrInterface): - print( - '\nYou are running a yield generator by polling the blockr.io website') - print( - 'This is quite bad for privacy. That site is owned by coinbase.com') - print( - 'Also your bot will run faster and more efficently, you can be immediately notified of new bitcoin network') - print( - ' information so your money will be working for you as hard as possible') - print( - 'Learn how to setup JoinMarket with Bitcoin Core: https://github.com/chris-belcher/joinmarket/wiki/Running-JoinMarket-with-Bitcoin-Core-full-node') - ret = raw_input('\nContinue? (y/n):') - if ret[0] != 'y': - return - - wallet = Wallet(seed, max_mix_depth=mix_levels) - jm_single().bc_interface.sync_wallet(wallet) - - jm_single().nickname = nickname - log.debug('starting yield generator') - irc = IRCMessageChannel(jm_single().nickname, - realname='btcint=' + jm_single().config.get( - "BLOCKCHAIN", "blockchain_source"), - password=nickserv_password) - maker = YieldGenerator(irc, wallet) - try: - log.debug('connecting to irc') - irc.run() - except: - log.debug('CRASHING, DUMPING EVERYTHING') - debug_dump_object(wallet, ['addr_cache', 'keys', 'seed']) - debug_dump_object(maker) - debug_dump_object(irc) - import traceback - log.debug(traceback.format_exc()) - - -if __name__ == "__main__": - main() - print('done') diff --git a/yield-generator-mixdepth.py b/yield-generator-mixdepth.py deleted file mode 100644 index 3e02800e..00000000 --- a/yield-generator-mixdepth.py +++ /dev/null @@ -1,328 +0,0 @@ -#! /usr/bin/env python -from __future__ import absolute_import, print_function - -import datetime -import os -import time -import binascii -import sys - -from joinmarket import Maker, IRCMessageChannel -from joinmarket import blockchaininterface, BlockrInterface -from joinmarket import jm_single, get_network, load_program_config -from joinmarket import random_nick -from joinmarket import get_log, calc_cj_fee, debug_dump_object -from joinmarket import Wallet - -#data_dir = os.path.dirname(os.path.realpath(__file__)) -#sys.path.insert(0, os.path.join(data_dir, 'lib')) - -#import bitcoin as btc -#import blockchaininterface - -from socket import gethostname -mix_levels = 5 - -#CONFIGURATION - -#miner fee contribution -txfee = 5000 -# fees for available mix levels from max to min amounts. -cjfee = ['0.00015', '0.00014', '0.00013', '0.00012', '0.00011'] -#cjfee = ["%0.5f" % (0.00015 - n*0.00001) for n in range(mix_levels)] -nickname = random_nick() -nickserv_password = '' - -#END CONFIGURATION -print(cjfee) -log = get_log() - - -#is a maker for the purposes of generating a yield from held -# bitcoins without ruining privacy for the taker, the taker could easily check -# the history of the utxos this bot sends, so theres not much incentive -# to ruin the privacy for barely any more yield -#sell-side algorithm: -#add up the value of each utxo for each mixing depth, -# announce a relative-fee order of the balance in each mixing depth -# amounts made to be non-overlapping -# minsize set by the miner fee contribution, so you never earn less in cjfee than miner fee -# cjfee drops as you go down to the lower-balance mixing depths, provides -# incentive for people to clump coins together for you in one mix depth -#announce an absolute fee order between the dust limit and minimum amount -# so that there is liquidity in the very low amounts too -class YieldGenerator(Maker): - statement_file = os.path.join('logs', 'yigen-statement.csv') - - def __init__(self, msgchan, wallet): - Maker.__init__(self, msgchan, wallet) - self.msgchan.register_channel_callbacks(self.on_welcome, - self.on_set_topic, None, None, - self.on_nick_leave, None) - self.tx_unconfirm_timestamp = {} - - def log_statement(self, data): - if get_network() == 'testnet': - return - - data = [str(d) for d in data] - self.income_statement = open(self.statement_file, 'a') - self.income_statement.write(','.join(data) + '\n') - self.income_statement.close() - - def on_welcome(self): - Maker.on_welcome(self) - if not os.path.isfile(self.statement_file): - self.log_statement( - ['timestamp', 'cj amount/satoshi', 'my input count', - 'my input value/satoshi', 'cjfee/satoshi', 'earned/satoshi', - 'confirm time/min', 'notes']) - - timestamp = datetime.datetime.now().strftime("%Y/%m/%d %H:%M:%S") - self.log_statement([timestamp, '', '', '', '', '', '', 'Connected']) - - def create_my_orders(self): - mix_balance = self.wallet.get_balance_by_mixdepth() - log.debug('mix_balance = ' + str(mix_balance)) - nondust_mix_balance = dict([(m, b) - for m, b in mix_balance.iteritems() - if b > jm_single().DUST_THRESHOLD]) - if len(nondust_mix_balance) == 0: - log.debug('do not have any coins left') - return [] - #sorts the mixdepth_balance map by balance size - sorted_mix_balance = sorted( - list(mix_balance.iteritems()), - key=lambda a: a[1], - reverse=True) - minsize = int( - 1.5 * txfee / float(min(cjfee)) - ) #minimum size is such that you always net profit at least 50% of the miner fee - filtered_mix_balance = [f for f in sorted_mix_balance if f[1] > minsize] - delta = mix_levels - len(filtered_mix_balance) - log.debug('minsize=' + str(minsize) + ' calc\'d with cjfee=' + str(min( - cjfee))) - lower_bound_balances = filtered_mix_balance[1:] + [(-1, minsize)] - mix_balance_min = [ - (mxb[0], mxb[1], minb[1]) - for mxb, minb in zip(filtered_mix_balance, lower_bound_balances) - ] - mix_balance_min = mix_balance_min[::-1] #reverse list order - thecjfee = cjfee[::-1] - - log.debug('mixdepth_balance_min = ' + str(mix_balance_min)) - orders = [] - oid = 0 - for mix_bal_min in mix_balance_min: - mixdepth, balance, mins = mix_bal_min - #the maker class reads specific keys from the dict, but others - # are allowed in there and will be ignored - order = {'oid': oid + 1, - 'ordertype': 'relorder', - 'minsize': max(mins - jm_single().DUST_THRESHOLD, - jm_single().DUST_THRESHOLD) + 1, - 'maxsize': max(balance - max(jm_single().DUST_THRESHOLD, txfee), - jm_single().DUST_THRESHOLD), - 'txfee': txfee, - 'cjfee': thecjfee[oid + delta], - 'mixdepth': mixdepth} - oid += 1 - orders.append(order) - - absorder_size = min(minsize, sorted_mix_balance[0][1]) - if absorder_size != 0: - lowest_cjfee = thecjfee[min(oid, len(thecjfee) - 1)] - absorder_fee = calc_cj_fee('relorder', lowest_cjfee, minsize) - log.debug('absorder fee = ' + str(absorder_fee) + ' uses cjfee=' + - str(lowest_cjfee)) - #the absorder is always oid=0 - order = {'oid': 0, - 'ordertype': 'absorder', - 'minsize': jm_single().DUST_THRESHOLD + 1, - 'maxsize': absorder_size - jm_single().DUST_THRESHOLD, - 'txfee': txfee, - 'cjfee': absorder_fee} - orders = [order] + orders - log.debug('generated orders = \n' + '\n'.join([str(o) for o in orders])) - - # sanity check - for order in orders: - assert order['minsize'] >= 0 - assert order['maxsize'] > 0 - assert order['minsize'] <= order['maxsize'] - - return orders - - def oid_to_order(self, cjorder, oid, amount): - total_amount = amount + cjorder.txfee - mix_balance = self.wallet.get_balance_by_mixdepth() - filtered_mix_balance = [m - for m in mix_balance.iteritems() - if m[1] >= total_amount] - if not filtered_mix_balance: - return None, None, None - log.debug('mix depths that have enough, filtered_mix_balance = ' + str( - filtered_mix_balance)) - - # use mix depth that has the closest amount of coins to what this transaction needs - # keeps coins moving through mix depths more quickly - # and its more likely to use txos of a similiar size to this transaction - filtered_mix_balance = sorted( - filtered_mix_balance, - key=lambda x: x[1]) #sort smallest to largest usable amount - - log.debug('sorted order of filtered_mix_balance = ' + str( - filtered_mix_balance)) - - mixdepth = filtered_mix_balance[0][0] - - log.debug('filling offer, mixdepth=' + str(mixdepth)) - - # mixdepth is the chosen depth we'll be spending from - cj_addr = self.wallet.get_internal_addr((mixdepth + 1) % - self.wallet.max_mix_depth) - change_addr = self.wallet.get_internal_addr(mixdepth) - - utxos = self.wallet.select_utxos(mixdepth, total_amount) - my_total_in = sum([va['value'] for va in utxos.values()]) - real_cjfee = calc_cj_fee(cjorder.ordertype, cjorder.cjfee, amount) - change_value = my_total_in - amount - cjorder.txfee + real_cjfee - if change_value <= jm_single().DUST_THRESHOLD: - log.debug('change value=%d below dust threshold, finding new utxos' - % (change_value)) - try: - utxos = self.wallet.select_utxos( - mixdepth, total_amount + jm_single().DUST_THRESHOLD) - except Exception: - log.debug( - 'dont have the required UTXOs to make a output above the dust threshold, quitting') - return None, None, None - - return utxos, cj_addr, change_addr - - def on_tx_unconfirmed(self, cjorder, txid, removed_utxos): - self.tx_unconfirm_timestamp[cjorder.cj_addr] = int(time.time()) - ''' - case 0 - the absorder will basically never get changed, unless there are no utxos left, when neworders==[] - case 1 - a single coin is split into two coins across levels - must announce a new order, plus modify the old order - case 2 - two existing mixdepths get modified - announce the modified new orders - case 3 - one existing mixdepth gets emptied into another - cancel it, modify the place it went - - algorithm - find all the orders which have changed, the length of that list tells us which case - ''' - - myorders = self.create_my_orders() - oldorderlist = self.orderlist - if len(myorders) == 0: - return ([o['oid'] for o in oldorderlist], []) - - cancel_orders = [] - ann_orders = [] - - neworders = [o for o in myorders if o['ordertype'] == 'relorder'] - oldorders = [o for o in oldorderlist if o['ordertype'] == 'relorder'] - #new_setdiff_old = The relative complement of `new` in `old` = members in `new` which are not in `old` - new_setdiff_old = [o for o in neworders if o not in oldorders] - old_setdiff_new = [o for o in oldorders if o not in neworders] - - log.debug('neworders = \n' + '\n'.join([str(o) for o in neworders])) - log.debug('oldorders = \n' + '\n'.join([str(o) for o in oldorders])) - log.debug('new_setdiff_old = \n' + '\n'.join([str( - o) for o in new_setdiff_old])) - log.debug('old_setdiff_new = \n' + '\n'.join([str( - o) for o in old_setdiff_new])) - if len(neworders) == len(oldorders): - ann_orders = new_setdiff_old - elif len(neworders) > len(oldorders): - ann_orders = new_setdiff_old - elif len(neworders) < len(oldorders): - ann_orders = new_setdiff_old - ann_oids = [o['oid'] for o in ann_orders] - cancel_orders = [o['oid'] - for o in old_setdiff_new - if o['oid'] not in ann_oids] - - #check if the absorder has changed, or if it needs to be newly announced - new_abs = [o for o in myorders if o['ordertype'] == 'absorder'] - old_abs = [o for o in oldorderlist if o['ordertype'] == 'absorder'] - if len(new_abs) > len(old_abs): - #announce an absorder where there wasnt one before - ann_orders = [new_abs[0]] + ann_orders - elif len(new_abs) == len(old_abs) and len(old_abs) > 0: - #maxsize is the only thing that changes, except cjfee but that changes at the same time - if new_abs[0]['maxsize'] != old_abs[0]['maxsize']: - ann_orders = [new_abs[0]] + ann_orders - - log.debug('can_orders = \n' + '\n'.join([str(o) for o in cancel_orders - ])) - log.debug('ann_orders = \n' + '\n'.join([str(o) for o in ann_orders])) - return (cancel_orders, ann_orders) - - def on_tx_confirmed(self, cjorder, confirmations, txid): - if cjorder.cj_addr in self.tx_unconfirm_timestamp: - confirm_time = int(time.time()) - self.tx_unconfirm_timestamp[ - cjorder.cj_addr] - del self.tx_unconfirm_timestamp[cjorder.cj_addr] - else: - confirm_time = 0 - timestamp = datetime.datetime.now().strftime("%Y/%m/%d %H:%M:%S") - self.log_statement([timestamp, cjorder.cj_amount, len( - cjorder.utxos), sum([av['value'] for av in cjorder.utxos.values( - )]), cjorder.real_cjfee, cjorder.real_cjfee - cjorder.txfee, round( - confirm_time / 60.0, 2), '']) - return self.on_tx_unconfirmed(cjorder, txid, None) - - -def main(): - load_program_config() - import sys - seed = sys.argv[1] - if isinstance(jm_single().bc_interface, - blockchaininterface.BlockrInterface): - print( - '\nYou are running a yield generator by polling the blockr.io website') - print( - 'This is quite bad for privacy. That site is owned by coinbase.com') - print( - 'Also your bot will run faster and more efficently, you can be immediately notified of new bitcoin network') - print( - ' information so your money will be working for you as hard as possible') - print( - 'Learn how to setup JoinMarket with Bitcoin Core: https://github.com/chris-belcher/joinmarket/wiki/Running-JoinMarket-with-Bitcoin-Core-full-node') - ret = raw_input('\nContinue? (y/n):') - if ret[0] != 'y': - return - - wallet = Wallet(seed, max_mix_depth=mix_levels) - jm_single().bc_interface.sync_wallet(wallet) - - jm_single().nickname = nickname - log.debug('starting yield generator') - irc = IRCMessageChannel(jm_single().nickname, - realname='btcint=' + jm_single().config.get( - "BLOCKCHAIN", "blockchain_source"), - password=nickserv_password) - maker = YieldGenerator(irc, wallet) - try: - log.debug('connecting to irc') - irc.run() - except: - log.debug('CRASHING, DUMPING EVERYTHING') - debug_dump_object(wallet, ['addr_cache', 'keys', 'seed']) - debug_dump_object(maker) - debug_dump_object(irc) - import traceback - log.debug(traceback.format_exc()) - - -if __name__ == "__main__": - main() - print('done') diff --git a/yield-generator-oscillator.py b/yield-generator-oscillator.py deleted file mode 100644 index 18ea381d..00000000 --- a/yield-generator-oscillator.py +++ /dev/null @@ -1,727 +0,0 @@ -#! /usr/bin/env python -from __future__ import absolute_import, print_function - -import os -import sys -import datetime -import time -import random -import decimal -import ConfigParser -import csv -import threading -import copy - -from joinmarket import Maker, IRCMessageChannel, OrderbookWatch -from joinmarket import blockchaininterface, BlockrInterface -from joinmarket import jm_single, get_network, load_program_config -from joinmarket import random_nick -from joinmarket import get_log, calc_cj_fee, debug_dump_object -from joinmarket import Wallet - -config = ConfigParser.RawConfigParser() -config.read('joinmarket.cfg') -mix_levels = 5 -nickname = random_nick() -nickserv_password = '' - -# EXPLANATION -# It watches your own transaction volume, and raises or lowers prices based -# on how many transactions you are getting. -# Price ranges that arent getting used will stay cheap, -# while frequently used price ranges will raise themselves in price. -# -# CONFIGURATION -# starting size: offer amount in btc -# price floor: cjfee in satoshis -# price increment: increase your cjfee by this much per tranaction -# time frame: Transaction count is within this window. This is how long it -# takes to drop to the price floor if there have been no -# transactions. A short window is more aggressive at dropping -# fees, while a large window holds prices up longer. -# type: absolute or relative - -offer_levels = ( - {'starting_size': 0, - 'type': 'absolute', - 'price_floor': 2000, - 'price_increment': 1000, # satoshis - 'price_ceiling': None, - 'time_frame': 7 * 24, - }, - {'starting_size': 1, - 'type': 'absolute', - 'price_floor': 3000, - 'price_increment': 1.25, # 25% - 'price_ceiling': None, - 'time_frame': 7 * 24, - }, - {'starting_size': 10, - 'type': 'relative', - 'price_floor': 5000, # type is relative, so 0.00005000 - 'price_increment': 1.618, # 61.8% is the fibonacci sequence/golden ratio - 'price_ceiling': None, - 'time_frame': 7 * 24, - }, -) - -# Here are randomizers you can use in your offers. -#'starting_size': random.uniform(0.9, 1), -#'starting_size': random.uniform(9, 10), -#'price_floor': random.randrange(1900, 2100), -#'price_floor': int(1000 * random.uniform(0.9, 1.1)), -#'price_increment': int(1000 * random.uniform(0.9, 1.1)), -#'type': random.choice(['absolute','relative']), -#'time_frame': random.randrange(6, 10) * 24, - -# tip: you can create a file called myoscoffers.py with your offer_levels in it. -# tip: view your csv file in unix with 'column -s, -t < yigen-statement-myfile.csv' - -# optional, for use in your joinmarket.cfg -""" -[YIELDGEN] -# Make your offer size never go above or below an amount -# offer low, default is output_size_min -# offer high, default is largest mix depth -# minimum output size, default is dust threshold 2730 satoshis -# multiple values means set randomly within that range -offer_low = 25000, 65000 -offer_high = 10e8, 14e8 -output_size_min = 10000 -""" - -#END CONFIGURATION - -try: - wallet_file = sys.argv[1] - statement_file = os.path.join( - 'logs', 'yigen-statement-' + wallet_file[:-5] + '.csv') -except: - sys.exit("You forgot to specify the wallet file.") - -try: - from myoscoffers import offer_levels -except: - pass - -try: - x = config.get('YIELDGEN', 'offer_low') - x = [r for r in csv.reader([x], skipinitialspace=True)][0] - if len(x) == 1: - offer_low = int(float(x[0])) - elif len(x) == 2: - offer_low = random.randrange( - int(float(x[0])), int(float(x[1]))) #random - elif len(x) > 2: - assert False -except (ConfigParser.NoOptionError, ConfigParser.NoSectionError) as e: - offer_low = None # will use output_min_size - -try: - x = config.get('YIELDGEN', 'offer_high') - x = [r for r in csv.reader([x], skipinitialspace=True)][0] - if len(x) == 1: - offer_high = int(float(x[0])) - elif len(x) == 2: - offer_high = random.randrange( - int(float(x[0])), int(float(x[1]))) #random - elif len(x) > 2: - assert False -except (ConfigParser.NoOptionError, ConfigParser.NoSectionError) as e: - offer_high = None # max mix depth will be used - -try: - x = config.get('YIELDGEN', 'output_size_min') - x = [r for r in csv.reader([x], skipinitialspace=True)][0] - if len(x) == 1: - output_size_min = int(float(x[0])) - elif len(x) == 2: - output_size_min = random.randrange( - int(float(x[0])), int(float(x[1]))) #random - elif len(x) > 2: - assert False -except (ConfigParser.NoOptionError, ConfigParser.NoSectionError) as e: - output_size_min = jm_single().DUST_THRESHOLD - -# the above config parser code could be moved into a library for reuse - -log = get_log() -log.debug(" ____ _ _ _ _ ") -log.debug(" / __ \ (_) | | | | ") -log.debug("| | | |___ ___ _| | | __ _| |_ ___ _ __ ") -log.debug("| | | / __|/ __| | | |/ _` | __/ _ \| '__|") -log.debug("| |__| \__ \ (__| | | | (_| | || (_) | | ") -log.debug(" \____/|___/\___|_|_|_|\__,_|\__\___/|_| ") -log.debug(random.choice([" yield generator for the civilized", - " the best yield generator there is", - " oscillator oscillator up and down"])) -if offer_low: - log.debug('offer_low = ' + str(offer_low) + " (" + str(offer_low / 1e8) + - " btc)") -if offer_high: - log.debug('offer_high = ' + str(offer_high) + " (" + str(offer_high / 1e8) + - " btc)") -else: - log.debug('offer_high = Max Mix Depth') -if output_size_min != jm_single().DUST_THRESHOLD: - log.debug('output_size_min = ' + str(output_size_min) + " (" + str( - output_size_min / 1e8) + " btc)") - - -def sanity_check(offers): - for offer in offers: - if offer['ordertype'] == 'absorder': - assert isinstance(offer['cjfee'], int) - elif offer['ordertype'] == 'relorder': - assert isinstance(offer['cjfee'], int) or isinstance(offer['cjfee'], - float) - assert offer['maxsize'] > 0 - assert offer['minsize'] > 0 - assert offer['minsize'] <= offer['maxsize'] - assert offer['txfee'] >= 0 - if offer_high: - assert offer['maxsize'] <= offer_high - assert (isinstance(offer['minsize'], int) or isinstance(offer['minsize'], long)) - assert (isinstance(offer['maxsize'], int) or isinstance(offer['maxsize'], long)) - assert isinstance(offer['txfee'], int) - assert offer['minsize'] >= offer_low - if offer['ordertype'] == 'absorder': - profit_max = offer['cjfee'] - offer['txfee'] - elif offer['ordertype'] == 'relorder': - profit_min = int(float(offer['cjfee']) * - offer['minsize']) - offer['txfee'] - profit_max = int(float(offer['cjfee']) * - offer['maxsize']) - offer['txfee'] - assert profit_min >= 0 - assert profit_max >= 0 - - -def offer_data_chart(offers): - has_rel = False - for offer in offers: - if offer['ordertype'] == 'relorder': - has_rel = True - offer_display = [] - header = 'oid'.rjust(4) - header += 'type'.rjust(5) - header += 'cjfee'.rjust(12) - header += 'minsize btc'.rjust(15) - header += 'maxsize btc'.rjust(15) - header += 'txfee'.rjust(7) - if has_rel: - header += 'minrev'.rjust(11) - header += 'maxrev'.rjust(11) - header += 'minprof'.rjust(11) - header += 'maxprof'.rjust(11) - else: - header += 'rev'.rjust(11) - header += 'prof'.rjust(11) - offer_display.append(header) - for offer in offers: - oid = str(offer['oid']) - if offer['ordertype'] == 'absorder': - ot = 'abs' - cjfee = str(offer['cjfee']) - minrev = '-' - maxrev = offer['cjfee'] - minprof = '-' - maxprof = int(maxrev - offer['txfee']) - elif offer['ordertype'] == 'relorder': - ot = 'rel' - cjfee = str('%.8f' % (offer['cjfee'] * 100)) - minrev = str(int(offer['cjfee'] * offer['minsize'])) - maxrev = str(int(offer['cjfee'] * offer['maxsize'])) - minprof = int(minrev) - offer['txfee'] - maxprof = int(maxrev) - offer['txfee'] - line = oid.rjust(4) - line += ot.rjust(5) - line += cjfee.rjust(12) - line += str('%.8f' % (offer['minsize'] / 1e8)).rjust(15) - line += str('%.8f' % (offer['maxsize'] / 1e8)).rjust(15) - line += str(offer['txfee']).rjust(7) - if has_rel: - line += str(minrev).rjust(11) - line += str(maxrev).rjust(11) - line += str(minprof).rjust(11) # minprof - line += str(maxprof).rjust(11) # maxprof - else: - line += str(maxrev).rjust(11) - line += str(maxprof).rjust(11) # maxprof - offer_display.append(line) - return offer_display - - -def get_recent_transactions(time_frame, show=False): - if not os.path.isfile(statement_file): - return [] - reader = csv.reader(open(statement_file, 'r')) - rows = [] - for row in reader: - rows.append(row) - rows = rows[1:] # remove heading - rows.reverse() - rows = sorted(rows, reverse=True) # just to be sure - xrows = [] - display_lines = [] - amount_total, earned_total = 0, 0 - for row in rows: - try: - timestamp = datetime.datetime.strptime(row[0], '%Y/%m/%d %H:%M:%S') - if timestamp < (datetime.datetime.now() - datetime.timedelta( - hours=time_frame)): - break - amount = int(row[1]) - my_input_count = int(row[2]) - my_input_value = int(row[3]) - cjfee = int(row[4]) # before txfee contrib - cjfee_earned = int(row[5]) - confirm_time = float(row[6]) - except ValueError: - continue - effective_rate = float('%.10f' % (cjfee_earned / float(amount))) # /0? - amount_total += amount - earned_total += cjfee_earned - xrows.append({'timestamp': timestamp, - 'amount': amount, - 'cjfee_earned': cjfee_earned, - 'confirm_time': confirm_time,}) - display_str = ' ' + timestamp.strftime("%Y-%m-%d %I:%M:%S %p") - display_str += str(float(confirm_time)).rjust(13) - display_str += str('%.8f' % (int(amount) / 1e8)).rjust(14) - display_str += str(int(cjfee_earned)).rjust(13) - display_str += str(('%.8f' % effective_rate) + ' %').rjust(16) - display_lines.append(display_str) - - if show and display_lines: - display = [ - ' datetime confirm min amount btc earned sat effectiverate' - ] - display = display + display_lines - display.append('-------------------------------------------'.rjust(79)) - total_effective_rate = float('%.10f' % - (earned_total / float(amount_total))) - ter_str = str(('%.8f' % total_effective_rate) + ' %').rjust(16) - display.append('Totals:'.rjust(36) + str('%.8f' % ( - amount_total / 1e8)).rjust(14) + str(earned_total).rjust(13) + - ter_str) - time_frame_days = time_frame / 24.0 - log.debug(str(len(xrows)) + ' transactions in the last ' + str( - time_frame) + ' hours: \n' + '\n'.join([str(x) for x in display])) - #log.debug(str(len(xrows)) + ' transactions in the last ' + str( - # time_frame) + ' hours (' + str(time_frame_days) + ' days) = \n' + - # '\n'.join([str(x) for x in display])) - elif show: - log.debug('No transactions in the last ' + str(time_frame) + ' hours.') - return xrows - - -def create_oscillator_offers(largest_mixdepth_size, sorted_mix_balance): - offer_lowx = max(offer_low, output_size_min) - if offer_high: - offer_highx = min(offer_high, largest_mixdepth_size - output_size_min) - else: - offer_highx = largest_mixdepth_size - output_size_min - offers = [] - display_lines = [] - oid = 0 - count = 0 - for offer in offer_levels: - count += 1 - lower = int(offer['starting_size'] * 1e8) - if lower < offer_lowx: - lower = offer_lowx - if count <= len(offer_levels) - 1: - upper = int((offer_levels[count]['starting_size'] * 1e8) - 1) - if upper > offer_highx: - upper = offer_highx - else: - upper = offer_highx - if lower > upper: - continue - fit_txs = [] - for tx in get_recent_transactions(offer['time_frame'], show=False): - if tx['amount'] >= lower and tx['amount'] <= upper: - fit_txs.append(tx) - amounts, earnings = [], [] - size_avg, earn_avg, effective_rate = 0, 0, 0 - if fit_txs: - amounts = [x['amount'] for x in fit_txs] - earnings = [x['cjfee_earned'] for x in fit_txs] - size_avg = sum(amounts) / len(amounts) - earn_avg = sum(earnings) / len(earnings) - effective_rate = float('%.10f' % - (sum(earnings) / float(sum(amounts)))) # /0? - if isinstance(offer['price_increment'], int): - tpi = offer['price_increment'] * len(fit_txs) - cjfee = offer['price_floor'] + tpi - elif isinstance(offer['price_increment'], float): - tpi = offer['price_increment']**len(fit_txs) - cjfee = int(round(offer['price_floor'] * tpi)) - else: - sys.exit('bad price_increment: ' + str(offer['price_increment'])) - if offer['price_ceiling'] and cjfee > offer['price_ceiling']: - cjfee = offer['price_ceiling'] - assert offer['type'] in ('absolute', 'relative') - if offer['type'] == 'absolute': - ordertype = 'absorder' - elif offer['type'] == 'relative': - ordertype = 'relorder' - cjfee = float('%.10f' % (cjfee / 1e10)) - oid += 1 - offerx = {'oid': oid, - 'ordertype': ordertype, - 'minsize': lower, - 'maxsize': upper, - 'txfee': 0, - 'cjfee': cjfee} - offers.append(offerx) - display_line = '' - display_line += str('%.8f' % (lower / 1e8)).rjust(15) - display_line += str('%.8f' % (upper / 1e8)).rjust(15) - display_line += str(offer['time_frame']).rjust(8) - display_line += str(len(fit_txs)).rjust(8) - display_line += str('%.8f' % (size_avg / 1e8)).rjust(15) - display_line += str(earn_avg).rjust(10) - display_line += str('%.8f' % (sum(amounts) / 1e8)).rjust(15) - display_line += str(sum(earnings)).rjust(10) - display_line += str('%.8f' % effective_rate).rjust(13) + ' %' - display_lines.append(display_line) - newoffers = [] - for offer in offers: - if not newoffers: - newoffers.append(offer) - continue - last_offer = copy.deepcopy(newoffers[-1]) - if (offer['minsize'] == last_offer['maxsize'] or \ - offer['minsize'] == last_offer['maxsize'] + 1) and \ - offer['cjfee'] == last_offer['cjfee']: - assert offer['txfee'] == last_offer['txfee'] - newoffers = newoffers[:-1] - last_offer['maxsize'] = offer['maxsize'] - newoffers.append(last_offer) - else: - newoffers.append(offer) - get_recent_transactions(24, show=True) - display = ['-------averages------- --------totals--------'.rjust(93)] - display.append( - ' minsize btc maxsize btc hours txs size btc ' + - 'earn sat size btc earn sat effectiverate') - log.debug('range summaries: \n' + '\n'.join([str( - x) for x in display + display_lines])) - log.debug('offer data chart: \n' + '\n'.join([str( - x) for x in offer_data_chart(offers)])) - if offers != newoffers: - #oid = 1 - #for offer in newoffers: - # offer['oid'] = oid - # oid += 1 - log.debug('final compressed offer data chart: \n' + '\n'.join([str( - x) for x in offer_data_chart(newoffers)])) - #log.debug('oscillator offers = \n' + '\n'.join([str(x) for x in offers])) - #log.debug('oscillator offers compressed = \n' + '\n'.join([str( - # o) for o in newoffers])) - return newoffers - - -class YieldGenerator(Maker, OrderbookWatch): - - def __init__(self, msgchan, wallet): - Maker.__init__(self, msgchan, wallet) - self.msgchan.register_channel_callbacks(self.on_welcome, - self.on_set_topic, None, None, - self.on_nick_leave, None) - self.tx_unconfirm_timestamp = {} - - def on_welcome(self): - Maker.on_welcome(self) - if not os.path.isfile(statement_file): - log.debug('Creating ' + str(statement_file)) - self.log_statement( - ['timestamp', 'cj amount/satoshi', 'my input count', - 'my input value/satoshi', 'cjfee/satoshi', 'earned/satoshi', - 'confirm time/min', 'notes']) - timestamp = datetime.datetime.now().strftime("%Y/%m/%d %H:%M:%S") - self.log_statement([timestamp, '', '', '', '', '', '', 'Connected']) - - def create_my_orders(self): - mix_balance = self.wallet.get_balance_by_mixdepth() - log.debug('mix_balance = ' + str(mix_balance)) - total_balance = 0 - for num, amount in mix_balance.iteritems(): - log.debug('for mixdepth=%d balance=%.8fbtc' % (num, amount / 1e8)) - total_balance += amount - log.debug('total balance = %.8fbtc' % (total_balance / 1e8)) - sorted_mix_balance = sorted( - list(mix_balance.iteritems()), - key=lambda a: a[1]) #sort by size - largest_mixdepth_size = sorted_mix_balance[-1][1] - if largest_mixdepth_size == 0: - print("ALERT: not enough funds available in wallet") - return [] - offers = create_oscillator_offers(largest_mixdepth_size, - sorted_mix_balance) - #log.debug('offer_data_chart = \n' + '\n'.join([str( - # x) for x in offer_data_chart(offers)])) - sanity_check(offers) - #log.debug('offers len = ' + str(len(offers))) - #log.debug('generated offers = \n' + '\n'.join([str(o) for o in offers])) - return offers - - def oid_to_order(self, cjorder, oid, amount): - '''Coins rotate circularly from max mixdepth back to mixdepth 0''' - mix_balance = self.wallet.get_balance_by_mixdepth() - total_amount = amount + cjorder.txfee - log.debug('amount, txfee, total_amount = ' + str(amount) + str( - cjorder.txfee) + str(total_amount)) - - # look for exact amount available with no change - # not supported because change output required - # needs this fixed https://github.com/JoinMarket-Org/joinmarket/issues/418 - #filtered_mix_balance = [m - # for m in mix_balance.iteritems() - # if m[1] == total_amount] - #if filtered_mix_balance: - # log.debug('mix depths that have the exact amount needed = ' + str( - # filtered_mix_balance)) - #else: - # log.debug('no mix depths contain the exact amount needed.') - - filtered_mix_balance = [m - for m in mix_balance.iteritems() - if m[1] >= (total_amount)] - log.debug('mix depths that have enough = ' + str(filtered_mix_balance)) - filtered_mix_balance = [m - for m in mix_balance.iteritems() - if m[1] >= total_amount + output_size_min] - log.debug('mix depths that have enough with output_size_min, ' + str( - filtered_mix_balance)) - - if not filtered_mix_balance: - log.debug('No mix depths have enough funds to cover the ' + - 'amount, cjfee, and output_size_min.') - return None, None, None - - # slinky clumping: push all coins towards the largest mixdepth, - # then spend from the largest mixdepth into the next mixdepth. - # the coins stay in the next mixdepth until they are all there, - # and then get spent into the next mixdepth, ad infinitum. - lmd = sorted(filtered_mix_balance, key=lambda x: x[1],)[-1] - smb = sorted(filtered_mix_balance, key=lambda x: x[0]) # seq of md num - mmd = self.wallet.max_mix_depth - nmd = (lmd[0] + 1) % mmd - if nmd not in [x[0] for x in smb]: # use all usable - next_si = (smb.index(lmd) + 1) % len(smb) - filtered_mix_balance = smb[next_si:] + smb[:next_si] - else: - nmd = [x for x in smb if x[0] == nmd][0] - others = [x for x in smb if x != nmd and x != lmd] - if not others: # just these two remain, prioritize largest - filtered_mix_balance = [lmd, nmd] - else: # use all usable - if [x for x in others if x[1] >= nmd[1]]: - next_si = (smb.index(lmd) + 1) % len(smb) - filtered_mix_balance = smb[next_si:] + smb[:next_si] - else: # others are not large, dont use nmd - next_si = (smb.index(lmd) + 2) % len(smb) - filtered_mix_balance = smb[next_si:] + smb[:next_si] - - # prioritize by mixdepths ascending - # keep coins moving towards last mixdepth, clumps there. - # makes sure coins sent to mixdepth 0 will get mixed to mixdepth 5 - #filtered_mix_balance = sorted(filtered_mix_balance, key=lambda x: x[0]) - - # use mix depth with the most coins, - # creates a more even distribution across mix depths - # and a more diverse txo selection in each depth - # sort largest to smallest amount - #filtered_mix_balance = sorted(filtered_mix_balance, key=lambda x: x[1], reverse=True) - - # use a random usable mixdepth. - # warning, could expose more txos to malicous taker requests - #filtered_mix_balance = [random.choice(filtered_mix_balance)] - - log.debug('sorted order of filtered_mix_balance = ' + str( - filtered_mix_balance)) - mixdepth = filtered_mix_balance[0][0] - log.debug('filling offer, mixdepth=' + str(mixdepth)) - # mixdepth is the chosen depth we'll be spending from - cj_addr = self.wallet.get_internal_addr((mixdepth + 1) % - self.wallet.max_mix_depth) - change_addr = self.wallet.get_internal_addr(mixdepth) - utxos = self.wallet.select_utxos(mixdepth, total_amount) - my_total_in = sum([va['value'] for va in utxos.values()]) - real_cjfee = calc_cj_fee(cjorder.ordertype, cjorder.cjfee, amount) - change_value = my_total_in - amount - cjorder.txfee + real_cjfee - if change_value <= output_size_min: - log.debug('change value=%d below dust threshold, finding new utxos' - % (change_value)) - try: - utxos = self.wallet.select_utxos(mixdepth, - total_amount + output_size_min) - except Exception: - log.debug( - 'dont have the required UTXOs to make a output above the dust threshold, quitting') - return None, None, None - return utxos, cj_addr, change_addr - - def refresh_offers(self): - cancel_orders, ann_orders = self.get_offer_diff() - self.modify_orders(cancel_orders, ann_orders) - - def get_offer_diff(self): - neworders = self.create_my_orders() - oldorders = self.orderlist - new_setdiff_old = [o for o in neworders if o not in oldorders] - old_setdiff_new = [o for o in oldorders if o not in neworders] - neworders = sorted(neworders, key=lambda x: x['oid']) - oldorders = sorted(oldorders, key=lambda x: x['oid']) - if neworders == oldorders: - log.debug('No orders modified for ' + nickname) - return ([], []) - """ - if neworders: - log.debug('neworders = \n' + '\n'.join([str(o) for o in neworders])) - if oldorders: - log.debug('oldorders = \n' + '\n'.join([str(o) for o in oldorders])) - if new_setdiff_old: - log.debug('new_setdiff_old = \n' + '\n'.join([str( - o) for o in new_setdiff_old])) - if old_setdiff_new: - log.debug('old_setdiff_new = \n' + '\n'.join([str( - o) for o in old_setdiff_new])) - """ - ann_orders = new_setdiff_old - ann_oids = [o['oid'] for o in ann_orders] - cancel_orders = [o['oid'] - for o in old_setdiff_new if o['oid'] not in ann_oids] - """ - if cancel_orders: - log.debug('can_orders = \n' + '\n'.join([str(o) for o in - cancel_orders])) - if ann_orders: - log.debug('ann_orders = \n' + '\n'.join([str(o) for o in ann_orders - ])) - """ - return (cancel_orders, ann_orders) - - def log_statement(self, data): - if get_network() == 'testnet': - return - data = [str(d) for d in data] - log.debug('Logging to ' + str(statement_file) + ': ' + str(data)) - assert len(data) == 8 - if data[7] == 'unconfirmed': # workaround - # on_tx_unconfirmed is being called by on_tx_confirmed - for row in csv.reader(open(statement_file, 'r')): - lastrow = row - if lastrow[1:6] == data[1:6]: - log.debug('Skipping double csv entry, workaround.') - pass - else: - fp = open(statement_file, 'a') - fp.write(','.join(data) + '\n') - fp.close() - elif data[7] != '': # 'Connected', 'notes' - fp = open(statement_file, 'a') - fp.write(','.join(data) + '\n') - fp.close() - else: # '' - rows = [] - for row in csv.reader(open(statement_file, 'r')): - rows.append(row) - fp = open(statement_file, 'w') - for row in rows: - if row[1:] == data[1:6] + ['0', 'unconfirmed']: - fp.write(','.join(data) + '\n') - log.debug('Found unconfirmed row, replacing.') - else: - fp.write(','.join(row) + '\n') - fp.close() - - def on_tx_unconfirmed(self, cjorder, txid, removed_utxos): - self.tx_unconfirm_timestamp[cjorder.cj_addr] = int(time.time()) - timestamp = datetime.datetime.now().strftime("%Y/%m/%d %H:%M:%S") - my_input_value = sum([av['value'] for av in cjorder.utxos.values()]) - earned = cjorder.real_cjfee - cjorder.txfee - self.log_statement([timestamp, cjorder.cj_amount, len( - cjorder.utxos), my_input_value, cjorder.real_cjfee, earned, '0', - 'unconfirmed']) - self.refresh_offers() # for oscillator - return self.get_offer_diff() - - def on_tx_confirmed(self, cjorder, confirmations, txid): - if cjorder.cj_addr in self.tx_unconfirm_timestamp: - confirm_time = int(time.time()) - self.tx_unconfirm_timestamp[ - cjorder.cj_addr] - confirm_time = round(confirm_time / 60.0, 2) - del self.tx_unconfirm_timestamp[cjorder.cj_addr] - else: - confirm_time = 0 - timestamp = datetime.datetime.now().strftime("%Y/%m/%d %H:%M:%S") - my_input_value = sum([av['value'] for av in cjorder.utxos.values()]) - earned = cjorder.real_cjfee - cjorder.txfee - self.log_statement([timestamp, cjorder.cj_amount, len( - cjorder.utxos), my_input_value, cjorder.real_cjfee, earned, - confirm_time, '']) - return self.on_tx_unconfirmed(cjorder, txid, None) - - -def main(): - load_program_config() - if isinstance(jm_single().bc_interface, - blockchaininterface.BlockrInterface): - print('You are using the blockr.io website') - print('You should setup JoinMarket with Bitcoin Core.') - ret = raw_input('\nContinue Anyways? (y/n):') - if ret[0] != 'y': - return - wallet = Wallet(wallet_file, max_mix_depth=mix_levels) - jm_single().bc_interface.sync_wallet(wallet) - jm_single().nickname = nickname - log.debug('starting yield generator') - irc = IRCMessageChannel(jm_single().nickname, - realname='btcint=' + jm_single().config.get( - "BLOCKCHAIN", "blockchain_source"), - password=nickserv_password) - maker = YieldGenerator(irc, wallet) - - def timer_loop(startup=False): # for oscillator - if not startup: - maker.refresh_offers() - poss_refresh = [] - for x in offer_levels: - recent_transactions = get_recent_transactions(x['time_frame']) - if recent_transactions: - oldest_transaction_time = recent_transactions[-1]['timestamp'] - else: - oldest_transaction_time = datetime.datetime.now() - next_refresh = oldest_transaction_time + datetime.timedelta( - hours=x['time_frame'], - seconds=1) - poss_refresh.append(next_refresh) - next_refresh = sorted(poss_refresh, key=lambda x: x)[0] - td = next_refresh - datetime.datetime.now() - seconds_till = (td.days * 24 * 60 * 60) + td.seconds - log.debug('Next offer refresh for ' + nickname + ' at ' + - next_refresh.strftime("%Y-%m-%d %I:%M:%S %p")) - log.debug('...or after a new transaction shows up.') - t = threading.Timer(seconds_till, timer_loop) - t.daemon = True - t.start() - - timer_loop(startup=True) - try: - log.debug('connecting to irc') - irc.run() - except: - log.debug('CRASHING, DUMPING EVERYTHING') - debug_dump_object(wallet, ['addr_cache', 'keys', 'seed']) - debug_dump_object(maker) - debug_dump_object(irc) - import traceback - log.debug(traceback.format_exc()) - - -if __name__ == "__main__": - main() - print('done')