From a3e18a6abaaa7382b4f4856256ebce1e68a0a359 Mon Sep 17 00:00:00 2001 From: Michael Schmoock Date: Thu, 20 Aug 2020 17:39:36 +0200 Subject: [PATCH 01/30] openchannel: make openchannel hook chainable This will make the `openchannel_hook` chainable. Logic is as follows: - The first plugin that rejects terminates the chain - If more than one plugin uses the `close_to` parameter, take the first value and log_unusual for the others. Changelog-Added: openchannel_hook is now chainable --- lightningd/opening_control.c | 143 ++++++++++++++++++++--------------- 1 file changed, 84 insertions(+), 59 deletions(-) diff --git a/lightningd/opening_control.c b/lightningd/opening_control.c index 8b554218c63f..0d96a9b8e15a 100644 --- a/lightningd/opening_control.c +++ b/lightningd/opening_control.c @@ -765,6 +765,8 @@ struct openchannel_hook_payload { u16 max_accepted_htlcs; u8 channel_flags; u8 *shutdown_scriptpubkey; + const u8 *our_upfront_shutdown_script; + char *errmsg; }; static void @@ -803,13 +805,12 @@ static void openchannel_payload_remove_openingd(struct subd *openingd, payload->openingd = NULL; } -static void openchannel_hook_cb(struct openchannel_hook_payload *payload STEALS, - const char *buffer, - const jsmntok_t *toks) +static void +openchannel_hook_final(struct openchannel_hook_payload *payload STEALS) { struct subd *openingd = payload->openingd; - const u8 *our_upfront_shutdown_script; - const char *errmsg = NULL; + const u8 *our_upfront_shutdown_script = payload->our_upfront_shutdown_script; + const char *errmsg = payload->errmsg; /* We want to free this, whatever happens. */ tal_steal(tmpctx, payload); @@ -820,65 +821,87 @@ static void openchannel_hook_cb(struct openchannel_hook_payload *payload STEALS, tal_del_destructor2(openingd, openchannel_payload_remove_openingd, payload); - /* If we had a hook, check what it says */ - if (buffer) { - const jsmntok_t *t = json_get_member(buffer, toks, "result"); - if (!t) - fatal("Plugin returned an invalid response to the" - " openchannel hook: %.*s", - toks[0].end - toks[0].start, - buffer + toks[0].start); - - if (json_tok_streq(buffer, t, "reject")) { - t = json_get_member(buffer, toks, "error_message"); - if (t) - errmsg = json_strdup(tmpctx, buffer, t); - else - errmsg = ""; - log_debug(openingd->ld->log, - "openchannel_hook_cb says '%s'", - errmsg); - our_upfront_shutdown_script = NULL; - } else if (!json_tok_streq(buffer, t, "continue")) - fatal("Plugin returned an invalid result for the " - "openchannel hook: %.*s", - t->end - t->start, buffer + t->start); - - /* Check for a 'close_to' address passed back */ - if (!errmsg) { - t = json_get_member(buffer, toks, "close_to"); - if (t) { - switch (json_to_address_scriptpubkey(tmpctx, chainparams, - buffer, t, - &our_upfront_shutdown_script)) { - case ADDRESS_PARSE_UNRECOGNIZED: - fatal("Plugin returned an invalid response to the" - " openchannel.close_to hook: %.*s", - t->end - t->start, buffer + t->start); - case ADDRESS_PARSE_WRONG_NETWORK: - fatal("Plugin returned invalid response to the" - " openchannel.close_to hook: address %s is" - " not on network %s", - tal_hex(NULL, our_upfront_shutdown_script), - chainparams->network_name); - case ADDRESS_PARSE_SUCCESS: - errmsg = NULL; - } - } else - our_upfront_shutdown_script = NULL; - } - } else - our_upfront_shutdown_script = NULL; - subd_send_msg(openingd, take(towire_openingd_got_offer_reply(NULL, errmsg, our_upfront_shutdown_script))); } -REGISTER_SINGLE_PLUGIN_HOOK(openchannel, - openchannel_hook_cb, - openchannel_hook_serialize, - struct openchannel_hook_payload *); +static bool +openchannel_hook_deserialize(struct openchannel_hook_payload *payload, + const char *buffer, + const jsmntok_t *toks) +{ + struct subd *openingd = payload->openingd; + + /* already rejected by prior plugin hook in the chain */ + if (payload->errmsg != NULL) + return true; + + if (!toks || !buffer) + return true; + + const jsmntok_t *t_result = json_get_member(buffer, toks, "result"); + const jsmntok_t *t_errmsg = json_get_member(buffer, toks, "error_message"); + const jsmntok_t *t_closeto = json_get_member(buffer, toks, "close_to"); + + if (!t_result) + fatal("Plugin returned an invalid response to the" + " openchannel hook: %.*s", + toks[0].end - toks[0].start, buffer + toks[0].start); + + /* reject */ + if (json_tok_streq(buffer, t_result, "reject")) { + payload->errmsg = ""; + if (t_errmsg) + payload->errmsg = json_strdup(payload, buffer, t_errmsg); + log_debug(openingd->ld->log, + "openchannel_hook rejects and says '%s'", + payload->errmsg); + if (t_closeto) + fatal("Plugin rejected openchannel but also set close_to"); + openchannel_hook_final(payload); + return false; + } else if (!json_tok_streq(buffer, t_result, "continue")) { + fatal("Plugin returned an invalid result for the " + "openchannel hook: %.*s", + t_result->end - t_result->start, buffer + t_result->start); + } + + /* Check for a valid 'close_to' address passed back */ + if (t_closeto) { + /* First plugin can set close_to. Log others. */ + if (payload->our_upfront_shutdown_script != NULL) { + log_unusual(openingd->ld->log, + "openchannel_hook close_to address was" + " already set by other plugin. Ignoring!"); + return true; + } + switch (json_to_address_scriptpubkey(tmpctx, chainparams, + buffer, t_closeto, + &payload->our_upfront_shutdown_script)) { + case ADDRESS_PARSE_UNRECOGNIZED: + fatal("Plugin returned an invalid response to" + " the openchannel.close_to hook: %.*s", + t_closeto->end - t_closeto->start, + buffer + t_closeto->start); + case ADDRESS_PARSE_WRONG_NETWORK: + fatal("Plugin returned invalid response to the" + " openchannel.close_to hook: address %s is" + " not on network %s", + tal_hex(NULL, payload->our_upfront_shutdown_script), + chainparams->network_name); + case ADDRESS_PARSE_SUCCESS: + break; + } + } + return true; +} + +REGISTER_PLUGIN_HOOK(openchannel, + openchannel_hook_deserialize, + openchannel_hook_final, + openchannel_hook_serialize, + struct openchannel_hook_payload *); static void opening_got_offer(struct subd *openingd, const u8 *msg, @@ -896,6 +919,8 @@ static void opening_got_offer(struct subd *openingd, payload = tal(openingd, struct openchannel_hook_payload); payload->openingd = openingd; + payload->our_upfront_shutdown_script = NULL; + payload->errmsg = NULL; if (!fromwire_openingd_got_offer(payload, msg, &payload->funding_satoshis, &payload->push_msat, From 2816c08036a56b4a98833315856967b7d5692050 Mon Sep 17 00:00:00 2001 From: Michael Schmoock Date: Fri, 21 Aug 2020 12:38:51 +0200 Subject: [PATCH 02/30] openchannel: test new hook chainable mechanics --- tests/plugins/accepter_close_to.py | 35 ----------- tests/plugins/openchannel_hook_accept.py | 19 ++++++ tests/plugins/openchannel_hook_accepter.py | 51 +++++++++++++++ tests/plugins/openchannel_hook_reject.py | 20 ++++++ tests/test_connection.py | 6 +- tests/test_plugin.py | 72 +++++++++++++++++++--- 6 files changed, 158 insertions(+), 45 deletions(-) delete mode 100755 tests/plugins/accepter_close_to.py create mode 100755 tests/plugins/openchannel_hook_accept.py create mode 100755 tests/plugins/openchannel_hook_accepter.py create mode 100755 tests/plugins/openchannel_hook_reject.py diff --git a/tests/plugins/accepter_close_to.py b/tests/plugins/accepter_close_to.py deleted file mode 100755 index 3711154d84f8..000000000000 --- a/tests/plugins/accepter_close_to.py +++ /dev/null @@ -1,35 +0,0 @@ -#!/usr/bin/env python3 -"""Simple plugin to test the openchannel_hook's - 'close_to' address functionality. - - If the funding amount is: - - a multiple of 11: we send back a valid address (regtest) - - a multiple of 7: we send back an empty address - - a multiple of 5: we send back an address for the wrong chain (mainnet) - - otherwise: we don't include the close_to -""" - -from pyln.client import Plugin, Millisatoshi - -plugin = Plugin() - - -@plugin.hook('openchannel') -def on_openchannel(openchannel, plugin, **kwargs): - # - a multiple of 11: we send back a valid address (regtest) - if Millisatoshi(openchannel['funding_satoshis']).to_satoshi() % 11 == 0: - return {'result': 'continue', 'close_to': 'bcrt1q7gtnxmlaly9vklvmfj06amfdef3rtnrdazdsvw'} - - # - a multiple of 7: we send back an empty address - if Millisatoshi(openchannel['funding_satoshis']).to_satoshi() % 7 == 0: - return {'result': 'continue', 'close_to': ''} - - # - a multiple of 5: we send back an address for the wrong chain (mainnet) - if Millisatoshi(openchannel['funding_satoshis']).to_satoshi() % 5 == 0: - return {'result': 'continue', 'close_to': 'bc1qlq8srqnz64wgklmqvurv7qnr4rvtq2u96hhfg2'} - - # - otherwise: we don't include the close_to - return {'result': 'continue'} - - -plugin.run() diff --git a/tests/plugins/openchannel_hook_accept.py b/tests/plugins/openchannel_hook_accept.py new file mode 100755 index 000000000000..afc922dc77c5 --- /dev/null +++ b/tests/plugins/openchannel_hook_accept.py @@ -0,0 +1,19 @@ +#!/usr/bin/env python3 +"""Plugin to test openchannel_hook + +Will simply accept any channel. Useful fot testing chained hook. +""" + +from pyln.client import Plugin + +plugin = Plugin() + + +@plugin.hook('openchannel') +def on_openchannel(openchannel, plugin, **kwargs): + msg = "accept on principle" + plugin.log(msg) + return {'result': 'continue'} + + +plugin.run() diff --git a/tests/plugins/openchannel_hook_accepter.py b/tests/plugins/openchannel_hook_accepter.py new file mode 100755 index 000000000000..5e7dec4ed857 --- /dev/null +++ b/tests/plugins/openchannel_hook_accepter.py @@ -0,0 +1,51 @@ +#!/usr/bin/env python3 +"""Simple plugin to test the openchannel_hook's + 'close_to' address functionality. + + If the funding amount is: + - 100005sat: we reject correctly w/o close_to + - 100004sat: we reject invalid by setting a close_to + - 100003sat: we send back a valid address (regtest) + - 100002sat: we send back an empty address + - 100001sat: we send back an address for the wrong chain (mainnet) + - otherwise: we don't include the close_to +""" + +from pyln.client import Plugin, Millisatoshi + +plugin = Plugin() + + +@plugin.hook('openchannel') +def on_openchannel(openchannel, plugin, **kwargs): + # - 100005sat: we reject correctly w/o close_to + if Millisatoshi(openchannel['funding_satoshis']).to_satoshi() == 100005: + msg = "reject for a reason" + plugin.log(msg) + return {'result': 'reject', 'error_message': msg} + + # - 100004sat: we reject invalid by setting a close_to + if Millisatoshi(openchannel['funding_satoshis']).to_satoshi() == 100004: + msg = "I am a broken plugin" + plugin.log(msg) + return {'result': 'reject', 'error_message': msg, + 'close_to': "bcrt1q7gtnxmlaly9vklvmfj06amfdef3rtnrdazdsvw"} + + # - 100003sat: we send back a valid address (regtest) + if Millisatoshi(openchannel['funding_satoshis']).to_satoshi() == 100003: + return {'result': 'continue', 'close_to': 'bcrt1q7gtnxmlaly9vklvmfj06amfdef3rtnrdazdsvw'} + + # - 100002sat: we send back an empty address + if Millisatoshi(openchannel['funding_satoshis']).to_satoshi() == 100002: + return {'result': 'continue', 'close_to': ''} + + # - 100001sat: we send back an address for the wrong chain (mainnet) + if Millisatoshi(openchannel['funding_satoshis']).to_satoshi() == 100001: + return {'result': 'continue', 'close_to': 'bc1qlq8srqnz64wgklmqvurv7qnr4rvtq2u96hhfg2'} + + # - otherwise: accept and don't include the close_to + plugin.log("accept by design") + return {'result': 'continue'} + + +plugin.run() diff --git a/tests/plugins/openchannel_hook_reject.py b/tests/plugins/openchannel_hook_reject.py new file mode 100755 index 000000000000..c89655dde2f3 --- /dev/null +++ b/tests/plugins/openchannel_hook_reject.py @@ -0,0 +1,20 @@ +#!/usr/bin/env python3 +"""Plugin to test openchannel_hook + +Will simply reject any channel with message "reject on principle". +Useful fot testing chained hook. +""" + +from pyln.client import Plugin + +plugin = Plugin() + + +@plugin.hook('openchannel') +def on_openchannel(openchannel, plugin, **kwargs): + msg = "reject on principle" + plugin.log(msg) + return {'result': 'reject', 'error_message': msg} + + +plugin.run() diff --git a/tests/test_connection.py b/tests/test_connection.py index d6b35aa2958b..f7ab73ffba1d 100644 --- a/tests/test_connection.py +++ b/tests/test_connection.py @@ -1050,13 +1050,13 @@ def test_funding_cancel_race(node_factory, bitcoind, executor): def test_funding_close_upfront(node_factory, bitcoind): l1 = node_factory.get_node() - opts = {'plugin': os.path.join(os.getcwd(), 'tests/plugins/accepter_close_to.py')} + opts = {'plugin': os.path.join(os.getcwd(), 'tests/plugins/openchannel_hook_accepter.py')} l2 = node_factory.get_node(options=opts) # The 'accepter_close_to' plugin uses the channel funding amount to determine # whether or not to include a 'close_to' address - amt_normal = 100007 # continues without returning a close_to - amt_addr = 100001 # returns valid regtest address + amt_normal = 100000 # continues without returning a close_to + amt_addr = 100003 # returns valid regtest address remote_valid_addr = 'bcrt1q7gtnxmlaly9vklvmfj06amfdef3rtnrdazdsvw' diff --git a/tests/test_plugin.py b/tests/test_plugin.py index b2772d5da29f..518f70311ba3 100644 --- a/tests/test_plugin.py +++ b/tests/test_plugin.py @@ -538,13 +538,7 @@ def test_openchannel_hook(node_factory, bitcoind): """ opts = [{}, {'plugin': os.path.join(os.getcwd(), 'tests/plugins/reject_odd_funding_amounts.py')}] l1, l2 = node_factory.line_graph(2, fundchannel=False, opts=opts) - - # Get some funds. - addr = l1.rpc.newaddr()['bech32'] - txid = bitcoind.rpc.sendtoaddress(addr, 10) - numfunds = len(l1.rpc.listfunds()['outputs']) - bitcoind.generate_block(1, txid) - wait_for(lambda: len(l1.rpc.listfunds()['outputs']) > numfunds) + l1.fundwallet(10**6) # Even amount: works. l1.rpc.fundchannel(l2.info['id'], 100000) @@ -574,6 +568,70 @@ def test_openchannel_hook(node_factory, bitcoind): l1.rpc.fundchannel(l2.info['id'], 100001) +def test_openchannel_hook_error_handling(node_factory, bitcoind): + """ l2 uses a plugin that should fatal() crash the node. + + This is because the plugin rejects a channel while + also setting a close_to address which isn't allowed. + """ + opts = {'plugin': os.path.join(os.getcwd(), 'tests/plugins/openchannel_hook_accepter.py')} + # openchannel_reject_but_set_close_to.py')} + l1 = node_factory.get_node() + l2 = node_factory.get_node(options=opts, + expect_fail=True, + may_fail=True, + allow_broken_log=True) + l1.connect(l2) + l1.fundwallet(10**6) + + # next fundchannel should fail fatal() for l2 + with pytest.raises(RpcError, match=r'Owning subdaemon openingd died'): + l1.rpc.fundchannel(l2.info['id'], 100004) + assert l2.daemon.is_in_log("Plugin rejected openchannel but also set close_to") + + +def test_openchannel_hook_chaining(node_factory, bitcoind): + """ l2 uses a set of plugin that all use the openchannel_hook. + + We test that chaining works by using multiple plugins in a way + that we check for the first plugin that rejects prevents from evaluating + further plugin responses down the chain. + + """ + opts = [{}, {'plugin': [ + os.path.join(os.path.dirname(__file__), '..', 'tests/plugins/openchannel_hook_accept.py'), + os.path.join(os.path.dirname(__file__), '..', 'tests/plugins/openchannel_hook_accepter.py'), + os.path.join(os.path.dirname(__file__), '..', 'tests/plugins/openchannel_hook_reject.py') + ]}] + l1, l2 = node_factory.line_graph(2, fundchannel=False, opts=opts) + l1.fundwallet(10**6) + + hook_msg = "openchannel_hook rejects and says '" + # 100005sat fundchannel should fail fatal() for l2 + # because hook_accepter.py rejects on that amount 'for a reason' + with pytest.raises(RpcError, match=r'They sent error channel'): + l1.rpc.fundchannel(l2.info['id'], 100005) + + # Note: hook chain order is currently undefined, because hooks are registerd + # as a result of the getmanifest call, which may take some random time. + # We need to workaround that fact, so test can be stable + correct_order = l2.daemon.is_in_log(hook_msg + "reject for a reason") + if correct_order: + assert l2.daemon.wait_for_log(hook_msg + "reject for a reason") + # the other plugin must not be called + assert not l2.daemon.is_in_log("reject on principle") + else: + assert l2.daemon.wait_for_log(hook_msg + "reject on principle") + # the other plugin must not be called + assert not l2.daemon.is_in_log("reject for a reason") + + # 100000sat is good for hook_accepter, so it should fail 'on principle' + # at third hook openchannel_reject.py + with pytest.raises(RpcError, match=r'They sent error channel'): + l1.rpc.fundchannel(l2.info['id'], 100000) + assert l2.daemon.wait_for_log(hook_msg + "reject on principle") + + @unittest.skipIf(not DEVELOPER, "without DEVELOPER=1, gossip v slow") def test_htlc_accepted_hook_fail(node_factory): """Send payments from l1 to l2, but l2 just declines everything. From d8e8426b52d9dd097c361e3cf405edbdfc20cffe Mon Sep 17 00:00:00 2001 From: Rusty Russell Date: Tue, 8 Sep 2020 06:24:26 +0930 Subject: [PATCH 03/30] Makefile: remove EXPERIMENTAL_FEATURES marker from generated files We're going to make experimental versions of these completely separate files. Also remove the dependency on the Makefile itself: it simply causes unnecessary churn. We can always force-rebuild when we change a rule. Signed-off-by: Rusty Russell --- Makefile | 23 ++++++++++------------- channeld/channeld_wiregen.c | 2 +- channeld/channeld_wiregen.h | 2 +- closingd/closingd_wiregen.c | 2 +- closingd/closingd_wiregen.h | 2 +- common/peer_status_wiregen.c | 2 +- common/peer_status_wiregen.h | 2 +- common/status_wiregen.c | 2 +- common/status_wiregen.h | 2 +- connectd/connectd_gossipd_wiregen.c | 2 +- connectd/connectd_gossipd_wiregen.h | 2 +- connectd/connectd_wiregen.c | 2 +- connectd/connectd_wiregen.h | 2 +- gossipd/gossip_store_wiregen.c | 2 +- gossipd/gossip_store_wiregen.h | 2 +- gossipd/gossipd_peerd_wiregen.c | 2 +- gossipd/gossipd_peerd_wiregen.h | 2 +- gossipd/gossipd_wiregen.c | 2 +- gossipd/gossipd_wiregen.h | 2 +- hsmd/hsmd_wiregen.c | 2 +- hsmd/hsmd_wiregen.h | 2 +- onchaind/onchaind_wiregen.c | 2 +- onchaind/onchaind_wiregen.h | 2 +- openingd/openingd_wiregen.c | 2 +- openingd/openingd_wiregen.h | 2 +- wire/common_wiregen.c | 2 +- wire/common_wiregen.h | 2 +- wire/onion_printgen.c | 2 +- wire/onion_printgen.h | 2 +- wire/onion_wiregen.c | 2 +- wire/onion_wiregen.h | 2 +- wire/peer_printgen.c | 2 +- wire/peer_printgen.h | 2 +- wire/peer_wiregen.c | 2 +- wire/peer_wiregen.h | 2 +- 35 files changed, 44 insertions(+), 47 deletions(-) diff --git a/Makefile b/Makefile index 5c791ea0a5b0..50049aa09ad0 100644 --- a/Makefile +++ b/Makefile @@ -201,8 +201,8 @@ CDUMP_OBJS := ccan-cdump.o ccan-strmap.o BOLT_GEN := tools/generate-wire.py WIRE_GEN := $(BOLT_GEN) -# If you use wiregen, you're dependent on the tool, its templates, and Makefile -WIRE_GEN_DEPS := $(WIRE_GEN) Makefile $(wildcard tools/gen/*_template) +# If you use wiregen, you're dependent on the tool and its templates +WIRE_GEN_DEPS := $(WIRE_GEN) $(wildcard tools/gen/*_template) # These are filled by individual Makefiles ALL_PROGRAMS := @@ -255,23 +255,23 @@ config.vars: # Git doesn't maintain timestamps, so we only regen if sources actually changed: # We place the SHA inside some generated files so we can tell if they need updating. -# Usage: $(call SHA256STAMP_CHANGED,prefix) -SHA256STAMP_CHANGED = [ x"`sed -n 's/.*SHA256STAMP://p' $@ 2>/dev/null`" != x"$(1)`cat $(filter-out FORCE,$^) | sha256sum | cut -c1-64`" ] -# Usage: $(call SHA256STAMP,prefix,commentprefix) -SHA256STAMP = echo '$(2) SHA256STAMP:$(1)'`cat $(filter-out FORCE,$^) | sha256sum | cut -c1-64` >> $@ +# Usage: $(call SHA256STAMP_CHANGED) +SHA256STAMP_CHANGED = [ x"`sed -n 's/.*SHA256STAMP://p' $@ 2>/dev/null`" != x"`cat $(filter-out FORCE,$^) | sha256sum | cut -c1-64`" ] +# Usage: $(call SHA256STAMP,commentprefix) +SHA256STAMP = echo '$(1) SHA256STAMP:'`cat $(filter-out FORCE,$^) | sha256sum | cut -c1-64` >> $@ # generate-wire.py --page [header|impl] hdrfilename wirename < csv > file %_wiregen.h: %_wire.csv $(WIRE_GEN_DEPS) - @if $(call SHA256STAMP_CHANGED,exp-$(EXPERIMENTAL_FEATURES)-); then $(call VERBOSE,"wiregen $@",tools/generate-wire.py --page header $($@_args) $@ `basename $< .csv` < $< > $@ && $(call SHA256STAMP,exp-$(EXPERIMENTAL_FEATURES)-,//)); fi + @if $(call SHA256STAMP_CHANGED); then $(call VERBOSE,"wiregen $@",tools/generate-wire.py --page header $($@_args) $@ `basename $< .csv` < $< > $@ && $(call SHA256STAMP,//)); fi %_wiregen.c: %_wire.csv $(WIRE_GEN_DEPS) - @if $(call SHA256STAMP_CHANGED,exp-$(EXPERIMENTAL_FEATURES)-); then $(call VERBOSE,"wiregen $@",tools/generate-wire.py --page impl $($@_args) ${@:.c=.h} `basename $< .csv` < $< > $@ && $(call SHA256STAMP,exp-$(EXPERIMENTAL_FEATURES)-,//)); fi + @if $(call SHA256STAMP_CHANGED); then $(call VERBOSE,"wiregen $@",tools/generate-wire.py --page impl $($@_args) ${@:.c=.h} `basename $< .csv` < $< > $@ && $(call SHA256STAMP,//)); fi %_printgen.h: %_wire.csv $(WIRE_GEN_DEPS) - @if $(call SHA256STAMP_CHANGED,exp-$(EXPERIMENTAL_FEATURES)-); then $(call VERBOSE,"printgen $@",tools/generate-wire.py -s -P --page header $($@_args) $@ `basename $< .csv` < $< > $@ && $(call SHA256STAMP,exp-$(EXPERIMENTAL_FEATURES)-,//)); fi + @if $(call SHA256STAMP_CHANGED); then $(call VERBOSE,"printgen $@",tools/generate-wire.py -s -P --page header $($@_args) $@ `basename $< .csv` < $< > $@ && $(call SHA256STAMP,//)); fi %_printgen.c: %_wire.csv $(WIRE_GEN_DEPS) - @if $(call SHA256STAMP_CHANGED,exp-$(EXPERIMENTAL_FEATURES)-); then $(call VERBOSE,"printgen $@",tools/generate-wire.py -s -P --page impl $($@_args) ${@:.c=.h} `basename $< .csv` < $< > $@ && $(call SHA256STAMP,exp-$(EXPERIMENTAL_FEATURES)-,//)); fi + @if $(call SHA256STAMP_CHANGED); then $(call VERBOSE,"printgen $@",tools/generate-wire.py -s -P --page impl $($@_args) ${@:.c=.h} `basename $< .csv` < $< > $@ && $(call SHA256STAMP,//)); fi include external/Makefile include bitcoin/Makefile @@ -478,9 +478,6 @@ $(CCAN_OBJS) $(CDUMP_OBJS): $(CCAN_HEADERS) Makefile # Except for CCAN, we treat everything else as dependent on external/ bitcoin/ common/ wire/ and all generated headers, and Makefile $(ALL_OBJS): $(BITCOIN_HEADERS) $(COMMON_HEADERS) $(CCAN_HEADERS) $(WIRE_HEADERS) $(ALL_GEN_HEADERS) $(EXTERNAL_HEADERS) Makefile -# Regen headers when Makefile changes -$(ALL_GEN_HEADERS): Makefile - update-ccan: mv ccan ccan.old DIR=$$(pwd)/ccan; cd ../ccan && ./tools/create-ccan-tree -a $$DIR `cd $$DIR.old/ccan && find * -name _info | sed s,/_info,, | $(SORT)` $(CCAN_NEW) diff --git a/channeld/channeld_wiregen.c b/channeld/channeld_wiregen.c index 1a8ffd12b579..f6465a8c073d 100644 --- a/channeld/channeld_wiregen.c +++ b/channeld/channeld_wiregen.c @@ -1185,4 +1185,4 @@ bool fromwire_send_onionmsg(const tal_t *ctx, const void *p, u8 onion[1366], str return cursor != NULL; } -// SHA256STAMP:exp-0-3a681f3ef41ab044c618653409c793a163a7c777181af6c798aff7bc9bef1a77 +// SHA256STAMP:b336984ad884507c4624dec512a7ac93b4bfe92d543a1e077a1d2cf7d3c30fd3 diff --git a/channeld/channeld_wiregen.h b/channeld/channeld_wiregen.h index 61da3dbf7eac..082d9381ed54 100644 --- a/channeld/channeld_wiregen.h +++ b/channeld/channeld_wiregen.h @@ -230,4 +230,4 @@ bool fromwire_send_onionmsg(const tal_t *ctx, const void *p, u8 onion[1366], str #endif /* LIGHTNING_CHANNELD_CHANNELD_WIREGEN_H */ -// SHA256STAMP:exp-0-3a681f3ef41ab044c618653409c793a163a7c777181af6c798aff7bc9bef1a77 +// SHA256STAMP:b336984ad884507c4624dec512a7ac93b4bfe92d543a1e077a1d2cf7d3c30fd3 diff --git a/closingd/closingd_wiregen.c b/closingd/closingd_wiregen.c index 23544b412d65..b3a9e1dfc885 100644 --- a/closingd/closingd_wiregen.c +++ b/closingd/closingd_wiregen.c @@ -200,4 +200,4 @@ bool fromwire_closingd_complete(const void *p) return cursor != NULL; } -// SHA256STAMP:exp-0-3235ada6c1c749bfb3885b20dae4c3a7deaa1b2716e34f30405a66cbe32d1055 +// SHA256STAMP:dbd3c58cd169d183d94e9b40e53243eb300e32d514f7891f332a117ff50f569c diff --git a/closingd/closingd_wiregen.h b/closingd/closingd_wiregen.h index 08a46f13cbae..9def7c7aeb54 100644 --- a/closingd/closingd_wiregen.h +++ b/closingd/closingd_wiregen.h @@ -56,4 +56,4 @@ bool fromwire_closingd_complete(const void *p); #endif /* LIGHTNING_CLOSINGD_CLOSINGD_WIREGEN_H */ -// SHA256STAMP:exp-0-3235ada6c1c749bfb3885b20dae4c3a7deaa1b2716e34f30405a66cbe32d1055 +// SHA256STAMP:dbd3c58cd169d183d94e9b40e53243eb300e32d514f7891f332a117ff50f569c diff --git a/common/peer_status_wiregen.c b/common/peer_status_wiregen.c index 210fc5e1c6ef..e54b01d13b3b 100644 --- a/common/peer_status_wiregen.c +++ b/common/peer_status_wiregen.c @@ -81,4 +81,4 @@ bool fromwire_status_peer_error(const tal_t *ctx, const void *p, struct channel_ return cursor != NULL; } -// SHA256STAMP:exp-0-e8a08a58cf4a3f67b0ad5f4cd690f4a6acb146644cd188ee5b9368451f45db70 +// SHA256STAMP:e956fdfdddc89d050229153e3db7a9539795b35b23d4c132f0f65eb0cc339d97 diff --git a/common/peer_status_wiregen.h b/common/peer_status_wiregen.h index d027d393be2a..1198d0fed8ed 100644 --- a/common/peer_status_wiregen.h +++ b/common/peer_status_wiregen.h @@ -35,4 +35,4 @@ bool fromwire_status_peer_error(const tal_t *ctx, const void *p, struct channel_ #endif /* LIGHTNING_COMMON_PEER_STATUS_WIREGEN_H */ -// SHA256STAMP:exp-0-e8a08a58cf4a3f67b0ad5f4cd690f4a6acb146644cd188ee5b9368451f45db70 +// SHA256STAMP:e956fdfdddc89d050229153e3db7a9539795b35b23d4c132f0f65eb0cc339d97 diff --git a/common/status_wiregen.c b/common/status_wiregen.c index 12046d7d319e..8978ba81816b 100644 --- a/common/status_wiregen.c +++ b/common/status_wiregen.c @@ -192,4 +192,4 @@ bool fromwire_status_peer_billboard(const tal_t *ctx, const void *p, bool *perm, return cursor != NULL; } -// SHA256STAMP:exp-0-7e7940a7f554e158b67dd56f142a1f8a6b9b22a968b5e8fe4e3e40e2dac1fced +// SHA256STAMP:8459550d3539dee0f13d3c40aa017972987cd7901131fdaa1be05d393424636c diff --git a/common/status_wiregen.h b/common/status_wiregen.h index cf293564ae2d..3ca1866ba1e3 100644 --- a/common/status_wiregen.h +++ b/common/status_wiregen.h @@ -54,4 +54,4 @@ bool fromwire_status_peer_billboard(const tal_t *ctx, const void *p, bool *perm, #endif /* LIGHTNING_COMMON_STATUS_WIREGEN_H */ -// SHA256STAMP:exp-0-7e7940a7f554e158b67dd56f142a1f8a6b9b22a968b5e8fe4e3e40e2dac1fced +// SHA256STAMP:8459550d3539dee0f13d3c40aa017972987cd7901131fdaa1be05d393424636c diff --git a/connectd/connectd_gossipd_wiregen.c b/connectd/connectd_gossipd_wiregen.c index 90e4e5d0769c..2bf116c7cef0 100644 --- a/connectd/connectd_gossipd_wiregen.c +++ b/connectd/connectd_gossipd_wiregen.c @@ -162,4 +162,4 @@ bool fromwire_gossipd_get_addrs_reply(const tal_t *ctx, const void *p, struct wi return cursor != NULL; } -// SHA256STAMP:exp-0-0da6e7838e67282265234c4f0a3576d32f3b10cda877991bf886a0bb2193cae7 +// SHA256STAMP:ff16f166eb5fe1a4389c503645deeea4b3e782cc3fb3fda7c87fab32c7f2ee05 diff --git a/connectd/connectd_gossipd_wiregen.h b/connectd/connectd_gossipd_wiregen.h index 52e674f09947..130c1a64e918 100644 --- a/connectd/connectd_gossipd_wiregen.h +++ b/connectd/connectd_gossipd_wiregen.h @@ -55,4 +55,4 @@ bool fromwire_gossipd_get_addrs_reply(const tal_t *ctx, const void *p, struct wi #endif /* LIGHTNING_CONNECTD_CONNECTD_GOSSIPD_WIREGEN_H */ -// SHA256STAMP:exp-0-0da6e7838e67282265234c4f0a3576d32f3b10cda877991bf886a0bb2193cae7 +// SHA256STAMP:ff16f166eb5fe1a4389c503645deeea4b3e782cc3fb3fda7c87fab32c7f2ee05 diff --git a/connectd/connectd_wiregen.c b/connectd/connectd_wiregen.c index 62da19dc0eff..69b84a5b4726 100644 --- a/connectd/connectd_wiregen.c +++ b/connectd/connectd_wiregen.c @@ -405,4 +405,4 @@ bool fromwire_connectd_dev_memleak_reply(const void *p, bool *leak) return cursor != NULL; } -// SHA256STAMP:exp-0-11e429feabbd0b87fd8c1bdb718b4edf20121eb67ef4ce2a3d8d24a37c258b53 +// SHA256STAMP:63529c4b1d95e8cb69c017829e3269e72c0d884545ae67b925f424b231c9633a diff --git a/connectd/connectd_wiregen.h b/connectd/connectd_wiregen.h index bb50ac59cc5f..0009653697ed 100644 --- a/connectd/connectd_wiregen.h +++ b/connectd/connectd_wiregen.h @@ -104,4 +104,4 @@ bool fromwire_connectd_dev_memleak_reply(const void *p, bool *leak); #endif /* LIGHTNING_CONNECTD_CONNECTD_WIREGEN_H */ -// SHA256STAMP:exp-0-11e429feabbd0b87fd8c1bdb718b4edf20121eb67ef4ce2a3d8d24a37c258b53 +// SHA256STAMP:63529c4b1d95e8cb69c017829e3269e72c0d884545ae67b925f424b231c9633a diff --git a/gossipd/gossip_store_wiregen.c b/gossipd/gossip_store_wiregen.c index 8b4740fd425d..e042260edbf6 100644 --- a/gossipd/gossip_store_wiregen.c +++ b/gossipd/gossip_store_wiregen.c @@ -118,4 +118,4 @@ bool fromwire_gossip_store_delete_chan(const void *p, struct short_channel_id *s return cursor != NULL; } -// SHA256STAMP:exp-0-e1b5e7cbfeabe064806c71200457814fbc6f987d6b49c7ba8f9d65df67dec958 +// SHA256STAMP:96e087792923799ac0e76d097bbf30972d627228742d69d9b1f90100cae8241a diff --git a/gossipd/gossip_store_wiregen.h b/gossipd/gossip_store_wiregen.h index e667bb7d0175..c621e16e2034 100644 --- a/gossipd/gossip_store_wiregen.h +++ b/gossipd/gossip_store_wiregen.h @@ -44,4 +44,4 @@ bool fromwire_gossip_store_delete_chan(const void *p, struct short_channel_id *s #endif /* LIGHTNING_GOSSIPD_GOSSIP_STORE_WIREGEN_H */ -// SHA256STAMP:exp-0-e1b5e7cbfeabe064806c71200457814fbc6f987d6b49c7ba8f9d65df67dec958 +// SHA256STAMP:96e087792923799ac0e76d097bbf30972d627228742d69d9b1f90100cae8241a diff --git a/gossipd/gossipd_peerd_wiregen.c b/gossipd/gossipd_peerd_wiregen.c index c29c667a089f..112754a3e8a8 100644 --- a/gossipd/gossipd_peerd_wiregen.c +++ b/gossipd/gossipd_peerd_wiregen.c @@ -227,4 +227,4 @@ bool fromwire_gossipd_local_channel_announcement(const tal_t *ctx, const void *p return cursor != NULL; } -// SHA256STAMP:exp-0-845eed7399afcb6681d6955638d216711dc10a151c37bfaea4de2d56b80be19d +// SHA256STAMP:8d8786f111a7dea8a44e064e382adf00265494b06cdf4cef1c6466bd03878b73 diff --git a/gossipd/gossipd_peerd_wiregen.h b/gossipd/gossipd_peerd_wiregen.h index 2182c8f51423..ba46fe7d7720 100644 --- a/gossipd/gossipd_peerd_wiregen.h +++ b/gossipd/gossipd_peerd_wiregen.h @@ -77,4 +77,4 @@ bool fromwire_gossipd_local_channel_announcement(const tal_t *ctx, const void *p #endif /* LIGHTNING_GOSSIPD_GOSSIPD_PEERD_WIREGEN_H */ -// SHA256STAMP:exp-0-845eed7399afcb6681d6955638d216711dc10a151c37bfaea4de2d56b80be19d +// SHA256STAMP:8d8786f111a7dea8a44e064e382adf00265494b06cdf4cef1c6466bd03878b73 diff --git a/gossipd/gossipd_wiregen.c b/gossipd/gossipd_wiregen.c index dd7de1c1b531..cf7992ea4a81 100644 --- a/gossipd/gossipd_wiregen.c +++ b/gossipd/gossipd_wiregen.c @@ -858,4 +858,4 @@ bool fromwire_gossipd_new_blockheight(const void *p, u32 *blockheight) return cursor != NULL; } -// SHA256STAMP:exp-0-4b29c599d2dfda038cce6193ef5b3b8e0e1000479608ff34743c77a0044f9cb4 +// SHA256STAMP:4aa62af856e3554603b0038eb0c9c6f5623cdb8d17abb6650338d1b728babd4b diff --git a/gossipd/gossipd_wiregen.h b/gossipd/gossipd_wiregen.h index 784af3e6766a..5361aa77bcf8 100644 --- a/gossipd/gossipd_wiregen.h +++ b/gossipd/gossipd_wiregen.h @@ -200,4 +200,4 @@ bool fromwire_gossipd_new_blockheight(const void *p, u32 *blockheight); #endif /* LIGHTNING_GOSSIPD_GOSSIPD_WIREGEN_H */ -// SHA256STAMP:exp-0-4b29c599d2dfda038cce6193ef5b3b8e0e1000479608ff34743c77a0044f9cb4 +// SHA256STAMP:4aa62af856e3554603b0038eb0c9c6f5623cdb8d17abb6650338d1b728babd4b diff --git a/hsmd/hsmd_wiregen.c b/hsmd/hsmd_wiregen.c index a0f6b3b829dd..4ae018fda418 100644 --- a/hsmd/hsmd_wiregen.c +++ b/hsmd/hsmd_wiregen.c @@ -1215,4 +1215,4 @@ bool fromwire_hsmd_get_output_scriptpubkey_reply(const tal_t *ctx, const void *p return cursor != NULL; } -// SHA256STAMP:exp-0-1fa355187cf25469af30fbb94920a8d0111620ec278b0dceaf2034a26b64f27b +// SHA256STAMP:a5ae76a1418b72d95fbb559fe568596318040de4e3726f2317969b9d9eee7788 diff --git a/hsmd/hsmd_wiregen.h b/hsmd/hsmd_wiregen.h index eec75e47a362..7618671c628a 100644 --- a/hsmd/hsmd_wiregen.h +++ b/hsmd/hsmd_wiregen.h @@ -272,4 +272,4 @@ bool fromwire_hsmd_get_output_scriptpubkey_reply(const tal_t *ctx, const void *p #endif /* LIGHTNING_HSMD_HSMD_WIREGEN_H */ -// SHA256STAMP:exp-0-1fa355187cf25469af30fbb94920a8d0111620ec278b0dceaf2034a26b64f27b +// SHA256STAMP:a5ae76a1418b72d95fbb559fe568596318040de4e3726f2317969b9d9eee7788 diff --git a/onchaind/onchaind_wiregen.c b/onchaind/onchaind_wiregen.c index 101408e48125..3acac3006217 100644 --- a/onchaind/onchaind_wiregen.c +++ b/onchaind/onchaind_wiregen.c @@ -627,4 +627,4 @@ bool fromwire_onchaind_notify_coin_mvt(const void *p, struct chain_coin_mvt *mvt return cursor != NULL; } -// SHA256STAMP:exp-0-a644f947dc012a0ea3c367ab99672f5b5a6233c8bf6fe4e3c607d57144d14063 +// SHA256STAMP:468582953e92150527baf350c80cdec02bed0a7a677fb87af0dcb4057236e00a diff --git a/onchaind/onchaind_wiregen.h b/onchaind/onchaind_wiregen.h index 09b0408e6a29..2b37dbaaca37 100644 --- a/onchaind/onchaind_wiregen.h +++ b/onchaind/onchaind_wiregen.h @@ -156,4 +156,4 @@ bool fromwire_onchaind_notify_coin_mvt(const void *p, struct chain_coin_mvt *mvt #endif /* LIGHTNING_ONCHAIND_ONCHAIND_WIREGEN_H */ -// SHA256STAMP:exp-0-a644f947dc012a0ea3c367ab99672f5b5a6233c8bf6fe4e3c607d57144d14063 +// SHA256STAMP:468582953e92150527baf350c80cdec02bed0a7a677fb87af0dcb4057236e00a diff --git a/openingd/openingd_wiregen.c b/openingd/openingd_wiregen.c index cb4e161c7c9e..2bb437465573 100644 --- a/openingd/openingd_wiregen.c +++ b/openingd/openingd_wiregen.c @@ -580,4 +580,4 @@ bool fromwire_openingd_dev_memleak_reply(const void *p, bool *leak) return cursor != NULL; } -// SHA256STAMP:exp-0-64ba9197a5a5fa3448590f0be252120d66e672626780227c48977ffad40a63a0 +// SHA256STAMP:164910f4bbe7967d43674158574280364e83db53c4487f5f470ac0d2a929f90b diff --git a/openingd/openingd_wiregen.h b/openingd/openingd_wiregen.h index 5ddba7e17653..08c9057dbc84 100644 --- a/openingd/openingd_wiregen.h +++ b/openingd/openingd_wiregen.h @@ -122,4 +122,4 @@ bool fromwire_openingd_dev_memleak_reply(const void *p, bool *leak); #endif /* LIGHTNING_OPENINGD_OPENINGD_WIREGEN_H */ -// SHA256STAMP:exp-0-64ba9197a5a5fa3448590f0be252120d66e672626780227c48977ffad40a63a0 +// SHA256STAMP:164910f4bbe7967d43674158574280364e83db53c4487f5f470ac0d2a929f90b diff --git a/wire/common_wiregen.c b/wire/common_wiregen.c index c5d0b835d897..ec0b1c5f2a18 100644 --- a/wire/common_wiregen.c +++ b/wire/common_wiregen.c @@ -101,4 +101,4 @@ bool fromwire_custommsg_out(const tal_t *ctx, const void *p, u8 **msg) return cursor != NULL; } -// SHA256STAMP:exp-0-239713127e6eda1db477804a5c4805e678384a9a4745c12cfa4bbd4212922b77 +// SHA256STAMP:b7c32f920bcd5725670e679935629442e3866d3cba8e149d3269fec6835c1908 diff --git a/wire/common_wiregen.h b/wire/common_wiregen.h index 2cd5ce763514..887525f9bf23 100644 --- a/wire/common_wiregen.h +++ b/wire/common_wiregen.h @@ -42,4 +42,4 @@ bool fromwire_custommsg_out(const tal_t *ctx, const void *p, u8 **msg); #endif /* LIGHTNING_WIRE_COMMON_WIREGEN_H */ -// SHA256STAMP:exp-0-239713127e6eda1db477804a5c4805e678384a9a4745c12cfa4bbd4212922b77 +// SHA256STAMP:b7c32f920bcd5725670e679935629442e3866d3cba8e149d3269fec6835c1908 diff --git a/wire/onion_printgen.c b/wire/onion_printgen.c index dd6d2f48c582..854ac894e303 100644 --- a/wire/onion_printgen.c +++ b/wire/onion_printgen.c @@ -654,4 +654,4 @@ void printonion_wire_tlv_message(const char *tlv_name, const u8 *msg) { } } -// SHA256STAMP:exp-0-ac5e129d4da055fa6973b64be133f59ec5b57e5b82216caa5cee2c182d732fa4 +// SHA256STAMP:ceeead04b6dc0928c189be726bd2ea88561db5e91e69ba610ef76ed1ba9e552e diff --git a/wire/onion_printgen.h b/wire/onion_printgen.h index a54e5d48f0f4..39ada48d74d9 100644 --- a/wire/onion_printgen.h +++ b/wire/onion_printgen.h @@ -58,4 +58,4 @@ void printwire_mpp_timeout(const char *fieldname, const u8 *cursor); #endif /* LIGHTNING_WIRE_ONION_PRINTGEN_H */ -// SHA256STAMP:exp-0-ac5e129d4da055fa6973b64be133f59ec5b57e5b82216caa5cee2c182d732fa4 +// SHA256STAMP:ceeead04b6dc0928c189be726bd2ea88561db5e91e69ba610ef76ed1ba9e552e diff --git a/wire/onion_wiregen.c b/wire/onion_wiregen.c index 715db8132acd..a09cef7f4d53 100644 --- a/wire/onion_wiregen.c +++ b/wire/onion_wiregen.c @@ -838,4 +838,4 @@ bool fromwire_mpp_timeout(const void *p) return cursor != NULL; } -// SHA256STAMP:exp-0-ac5e129d4da055fa6973b64be133f59ec5b57e5b82216caa5cee2c182d732fa4 +// SHA256STAMP:ceeead04b6dc0928c189be726bd2ea88561db5e91e69ba610ef76ed1ba9e552e diff --git a/wire/onion_wiregen.h b/wire/onion_wiregen.h index 66d829fd9bfe..99f597795136 100644 --- a/wire/onion_wiregen.h +++ b/wire/onion_wiregen.h @@ -208,4 +208,4 @@ bool fromwire_mpp_timeout(const void *p); #endif /* LIGHTNING_WIRE_ONION_WIREGEN_H */ -// SHA256STAMP:exp-0-ac5e129d4da055fa6973b64be133f59ec5b57e5b82216caa5cee2c182d732fa4 +// SHA256STAMP:ceeead04b6dc0928c189be726bd2ea88561db5e91e69ba610ef76ed1ba9e552e diff --git a/wire/peer_printgen.c b/wire/peer_printgen.c index 616bf388a6c7..f872e109f76f 100644 --- a/wire/peer_printgen.c +++ b/wire/peer_printgen.c @@ -2037,4 +2037,4 @@ void printpeer_wire_tlv_message(const char *tlv_name, const u8 *msg) { } } -// SHA256STAMP:exp-0-d5b13db56b4ff42a2866aeb87a3922ceb703b145a1a1be322f7157cac99d7b96 +// SHA256STAMP:387ba235bbb8248b61a18b8fc1b7842f5ad47bbbb53cc8cf2bb2b8f872ce5a0d diff --git a/wire/peer_printgen.h b/wire/peer_printgen.h index 183c73e0300d..ec1fc4d82b57 100644 --- a/wire/peer_printgen.h +++ b/wire/peer_printgen.h @@ -71,4 +71,4 @@ void printwire_channel_update_checksums(const char *fieldname, const u8 **cursor void printwire_channel_update_timestamps(const char *fieldname, const u8 **cursor, size_t *plen); #endif /* LIGHTNING_WIRE_PEER_PRINTGEN_H */ -// SHA256STAMP:exp-0-d5b13db56b4ff42a2866aeb87a3922ceb703b145a1a1be322f7157cac99d7b96 +// SHA256STAMP:387ba235bbb8248b61a18b8fc1b7842f5ad47bbbb53cc8cf2bb2b8f872ce5a0d diff --git a/wire/peer_wiregen.c b/wire/peer_wiregen.c index 552686500bf1..205e7c4198cd 100644 --- a/wire/peer_wiregen.c +++ b/wire/peer_wiregen.c @@ -2751,4 +2751,4 @@ bool fromwire_channel_update_option_channel_htlc_max(const void *p, secp256k1_ec return cursor != NULL; } -// SHA256STAMP:exp-0-d5b13db56b4ff42a2866aeb87a3922ceb703b145a1a1be322f7157cac99d7b96 +// SHA256STAMP:387ba235bbb8248b61a18b8fc1b7842f5ad47bbbb53cc8cf2bb2b8f872ce5a0d diff --git a/wire/peer_wiregen.h b/wire/peer_wiregen.h index 49376ac5c240..d978b4153ec4 100644 --- a/wire/peer_wiregen.h +++ b/wire/peer_wiregen.h @@ -596,4 +596,4 @@ bool fromwire_channel_update_option_channel_htlc_max(const void *p, secp256k1_ec #endif /* LIGHTNING_WIRE_PEER_WIREGEN_H */ -// SHA256STAMP:exp-0-d5b13db56b4ff42a2866aeb87a3922ceb703b145a1a1be322f7157cac99d7b96 +// SHA256STAMP:387ba235bbb8248b61a18b8fc1b7842f5ad47bbbb53cc8cf2bb2b8f872ce5a0d From c34c055d82a90d77cb13f10afd4168b791e6cb7f Mon Sep 17 00:00:00 2001 From: Rusty Russell Date: Tue, 8 Sep 2020 06:36:50 +0930 Subject: [PATCH 04/30] Makefile: use completely separate spec-derived files for EXPERIMENTAL_FEATURES This avoids overwriting the ones in git, and generally makes things neater. We have convenience headers wire/peer_wire.h and wire/onion_wire.h to avoid most #ifdefs: simply include those. Signed-off-by: Rusty Russell --- Makefile | 16 ++++++-- channeld/channeld.c | 2 +- channeld/channeld_htlc.h | 2 +- common/decode_array.c | 2 +- common/dev_disconnect.c | 2 +- common/gossip_rcvd_filter.c | 2 +- common/gossip_store.c | 2 +- common/gossmap.c | 2 +- common/htlc_wire.h | 2 +- common/onion.c | 2 +- common/ping.c | 2 +- common/sphinx.h | 2 +- common/test/Makefile | 4 +- common/wire_error.c | 2 +- connectd/Makefile | 2 +- connectd/connectd.c | 1 - devtools/Makefile | 6 +-- devtools/create-gossipstore.c | 2 +- devtools/decodemsg.c | 5 +++ devtools/dump-gossipstore.c | 2 +- devtools/mkgossip.c | 2 +- devtools/mkquery.c | 2 +- devtools/print_wire.h | 4 +- gossipd/Makefile | 2 +- gossipd/gossip_generation.c | 2 +- gossipd/gossip_store.c | 2 +- gossipd/gossipd.c | 2 +- gossipd/gossipd_wire.csv | 2 +- gossipd/gossipd_wiregen.c | 2 +- gossipd/gossipd_wiregen.h | 4 +- gossipd/queries.c | 2 +- gossipd/routing.c | 2 +- gossipd/routing.h | 2 +- gossipd/seeker.c | 2 +- gossipd/test/Makefile | 2 +- hsmd/hsmd.c | 2 +- lightningd/connect_control.c | 2 +- lightningd/gossip_control.c | 2 +- lightningd/htlc_end.h | 2 +- lightningd/htlc_set.h | 2 +- lightningd/invoice.h | 2 +- lightningd/notification.h | 2 +- lightningd/peer_control.c | 2 +- lightningd/peer_htlcs.c | 2 +- openingd/openingd.c | 1 - plugins/Makefile | 4 +- plugins/keysend.c | 2 +- plugins/libplugin-pay.h | 2 +- tools/test/Makefile | 2 +- wallet/wallet.h | 2 +- wire/Makefile | 37 +++++++++---------- wire/onion_wire.h | 12 ++++++ ...xtracted_onion_wire_csv => onion_wire_csv} | 0 wire/peer_wire.h | 5 +++ ...{extracted_peer_wire_csv => peer_wire_csv} | 0 wire/test/Makefile | 2 +- wire/test/run-peer-wire.c | 2 +- wire/test/run-tlvstream.c | 4 ++ 58 files changed, 109 insertions(+), 80 deletions(-) create mode 100644 wire/onion_wire.h rename wire/{extracted_onion_wire_csv => onion_wire_csv} (100%) rename wire/{extracted_peer_wire_csv => peer_wire_csv} (100%) diff --git a/Makefile b/Makefile index 50049aa09ad0..7bdbc0119704 100644 --- a/Makefile +++ b/Makefile @@ -253,6 +253,14 @@ config.vars: %.o: %.c @$(call VERBOSE, "cc $<", $(CC) $(CFLAGS) -c -o $@ $<) +# '_exp' inserted before _wiregen.[ch] to demark experimental +# spec-derived headers, which are *not* committed into git. +ifeq ($(EXPERIMENTAL_FEATURES),1) +EXP := _exp +else +EXP := +endif + # Git doesn't maintain timestamps, so we only regen if sources actually changed: # We place the SHA inside some generated files so we can tell if they need updating. # Usage: $(call SHA256STAMP_CHANGED) @@ -262,16 +270,16 @@ SHA256STAMP = echo '$(1) SHA256STAMP:'`cat $(filter-out FORCE,$^) | sha256sum | # generate-wire.py --page [header|impl] hdrfilename wirename < csv > file %_wiregen.h: %_wire.csv $(WIRE_GEN_DEPS) - @if $(call SHA256STAMP_CHANGED); then $(call VERBOSE,"wiregen $@",tools/generate-wire.py --page header $($@_args) $@ `basename $< .csv` < $< > $@ && $(call SHA256STAMP,//)); fi + @if $(call SHA256STAMP_CHANGED); then $(call VERBOSE,"wiregen $@",tools/generate-wire.py --page header $($@_args) $@ `basename $< .csv | sed 's/_exp_/_/'` < $< > $@ && $(call SHA256STAMP,//)); fi %_wiregen.c: %_wire.csv $(WIRE_GEN_DEPS) - @if $(call SHA256STAMP_CHANGED); then $(call VERBOSE,"wiregen $@",tools/generate-wire.py --page impl $($@_args) ${@:.c=.h} `basename $< .csv` < $< > $@ && $(call SHA256STAMP,//)); fi + @if $(call SHA256STAMP_CHANGED); then $(call VERBOSE,"wiregen $@",tools/generate-wire.py --page impl $($@_args) ${@:.c=.h} `basename $< .csv | sed 's/_exp_/_/'` < $< > $@ && $(call SHA256STAMP,//)); fi %_printgen.h: %_wire.csv $(WIRE_GEN_DEPS) - @if $(call SHA256STAMP_CHANGED); then $(call VERBOSE,"printgen $@",tools/generate-wire.py -s -P --page header $($@_args) $@ `basename $< .csv` < $< > $@ && $(call SHA256STAMP,//)); fi + @if $(call SHA256STAMP_CHANGED); then $(call VERBOSE,"printgen $@",tools/generate-wire.py -s -P --page header $($@_args) $@ `basename $< .csv | sed 's/_exp_/_/'` < $< > $@ && $(call SHA256STAMP,//)); fi %_printgen.c: %_wire.csv $(WIRE_GEN_DEPS) - @if $(call SHA256STAMP_CHANGED); then $(call VERBOSE,"printgen $@",tools/generate-wire.py -s -P --page impl $($@_args) ${@:.c=.h} `basename $< .csv` < $< > $@ && $(call SHA256STAMP,//)); fi + @if $(call SHA256STAMP_CHANGED); then $(call VERBOSE,"printgen $@",tools/generate-wire.py -s -P --page impl $($@_args) ${@:.c=.h} `basename $< .csv | sed 's/_exp_/_/'` < $< > $@ && $(call SHA256STAMP,//)); fi include external/Makefile include bitcoin/Makefile diff --git a/channeld/channeld.c b/channeld/channeld.c index 07f48f8987a5..95d8df756b7c 100644 --- a/channeld/channeld.c +++ b/channeld/channeld.c @@ -63,7 +63,7 @@ #include #include #include -#include +#include #include #include #include diff --git a/channeld/channeld_htlc.h b/channeld/channeld_htlc.h index 5913a500e4ac..118d9321f7d5 100644 --- a/channeld/channeld_htlc.h +++ b/channeld/channeld_htlc.h @@ -6,7 +6,7 @@ #include #include #include -#include +#include struct htlc { /* What's the status. */ diff --git a/common/decode_array.c b/common/decode_array.c index 37e482618284..f2da3018d06a 100644 --- a/common/decode_array.c +++ b/common/decode_array.c @@ -1,7 +1,7 @@ #include #include #include -#include +#include #include #include diff --git a/common/dev_disconnect.c b/common/dev_disconnect.c index 13cfa3c79c72..b3d7c07efeae 100644 --- a/common/dev_disconnect.c +++ b/common/dev_disconnect.c @@ -11,7 +11,7 @@ #include #include #include -#include +#include #if DEVELOPER /* We move the fd if and only if we do a disconnect. */ diff --git a/common/gossip_rcvd_filter.c b/common/gossip_rcvd_filter.c index 94f8ed4b7414..05f0dae77160 100644 --- a/common/gossip_rcvd_filter.c +++ b/common/gossip_rcvd_filter.c @@ -4,7 +4,7 @@ #include #include #include -#include +#include static u64 msg_key(const u8 *msg) { diff --git a/common/gossip_store.c b/common/gossip_store.c index 89a1c69d0fa4..4d49bf396fd2 100644 --- a/common/gossip_store.c +++ b/common/gossip_store.c @@ -9,7 +9,7 @@ #include #include #include -#include +#include void gossip_setup_timestamp_filter(struct per_peer_state *pps, u32 first_timestamp, diff --git a/common/gossmap.c b/common/gossmap.c index 1f51ebd61ad8..ee46abae6399 100644 --- a/common/gossmap.c +++ b/common/gossmap.c @@ -20,7 +20,7 @@ #include #include #include -#include +#include /* We need this global to decode indexes for hash functions */ static struct gossmap *map; diff --git a/common/htlc_wire.h b/common/htlc_wire.h index 743a743fbd8b..e5d085e13832 100644 --- a/common/htlc_wire.h +++ b/common/htlc_wire.h @@ -6,7 +6,7 @@ #include #include #include -#include +#include struct bitcoin_tx; struct shachain; diff --git a/common/onion.c b/common/onion.c index bd82eea7b2e1..acb8b8ad7942 100644 --- a/common/onion.c +++ b/common/onion.c @@ -5,7 +5,7 @@ #include #include #include -#include +#include /* BOLT #4: * diff --git a/common/ping.c b/common/ping.c index 5aed8667054e..8322e55a51ae 100644 --- a/common/ping.c +++ b/common/ping.c @@ -1,7 +1,7 @@ #include #include #include -#include +#include bool check_ping_make_pong(const tal_t *ctx, const u8 *ping, u8 **pong) { diff --git a/common/sphinx.h b/common/sphinx.h index 18f924a7fe2e..daad044e0371 100644 --- a/common/sphinx.h +++ b/common/sphinx.h @@ -10,7 +10,7 @@ #include #include #include -#include +#include struct node_id; diff --git a/common/test/Makefile b/common/test/Makefile index dfabba4d6290..42065c66960a 100644 --- a/common/test/Makefile +++ b/common/test/Makefile @@ -13,7 +13,7 @@ ALL_C_SOURCES += $(COMMON_TEST_SRC) ALL_TEST_PROGRAMS += $(COMMON_TEST_PROGRAMS) # Sphinx test wants to decode TLVs. -common/test/run-sphinx: wire/onion_wiregen.o wire/towire.o wire/fromwire.o +common/test/run-sphinx: wire/onion$(EXP)_wiregen.o wire/towire.o wire/fromwire.o common/test/run-param \ common/test/run-json: \ @@ -25,7 +25,7 @@ common/test/run-json: \ common/wireaddr.o \ common/type_to_string.o \ wire/fromwire.o \ - wire/onion_wiregen.o \ + wire/onion$(EXP)_wiregen.o \ wire/towire.o update-mocks: $(COMMON_TEST_SRC:%=update-mocks/%) diff --git a/common/wire_error.c b/common/wire_error.c index 537c51f025d7..67f6b773f295 100644 --- a/common/wire_error.c +++ b/common/wire_error.c @@ -3,7 +3,7 @@ #include #include #include -#include +#include u8 *towire_errorfmtv(const tal_t *ctx, const struct channel_id *channel, diff --git a/connectd/Makefile b/connectd/Makefile index 5b4e34327fd0..27cec25bb6de 100644 --- a/connectd/Makefile +++ b/connectd/Makefile @@ -63,7 +63,7 @@ CONNECTD_COMMON_OBJS := \ common/wire_error.o \ gossipd/gossipd_wiregen.o \ lightningd/gossip_msg.o \ - wire/onion_wiregen.o + wire/onion$(EXP)_wiregen.o lightningd/lightning_connectd: $(CONNECTD_OBJS) $(CONNECTD_COMMON_OBJS) $(BITCOIN_OBJS) $(WIRE_OBJS) $(HSMD_CLIENT_OBJS) diff --git a/connectd/connectd.c b/connectd/connectd.c index 1fcd9414ddc1..069a73436797 100644 --- a/connectd/connectd.c +++ b/connectd/connectd.c @@ -68,7 +68,6 @@ #include #include #include -#include #include #include #include diff --git a/devtools/Makefile b/devtools/Makefile index ade001e35e0b..d55ee1616113 100644 --- a/devtools/Makefile +++ b/devtools/Makefile @@ -41,8 +41,8 @@ DEVTOOLS_COMMON_OBJS := \ common/utils.o \ common/version.o \ common/wireaddr.o \ - wire/onion_wiregen.o \ - wire/peer_wiregen.o + wire/onion$(EXP)_wiregen.o \ + wire/peer$(EXP)_wiregen.o devtools/bolt11-cli: $(DEVTOOLS_COMMON_OBJS) $(JSMN_OBJS) $(CCAN_OBJS) $(BITCOIN_OBJS) wire/fromwire.o wire/towire.o devtools/bolt11-cli.o @@ -61,7 +61,7 @@ devtools/onion: $(DEVTOOLS_COMMON_OBJS) $(JSMN_OBJS) $(CCAN_OBJS) $(BITCOIN_OBJS devtools/blindedpath: $(DEVTOOLS_COMMON_OBJS) $(JSMN_OBJS) $(CCAN_OBJS) common/blinding.o $(BITCOIN_OBJS) wire/fromwire.o wire/towire.o devtools/blindedpath.o common/onion.o common/onionreply.o common/sphinx.o -devtools/gossipwith: $(DEVTOOLS_COMMON_OBJS) $(JSMN_OBJS) $(CCAN_OBJS) $(BITCOIN_OBJS) wire/fromwire.o wire/towire.o wire/peer_wiregen.o devtools/gossipwith.o common/cryptomsg.o common/cryptomsg.o common/crypto_sync.o +devtools/gossipwith: $(DEVTOOLS_COMMON_OBJS) $(JSMN_OBJS) $(CCAN_OBJS) $(BITCOIN_OBJS) wire/fromwire.o wire/towire.o wire/peer$(EXP)_wiregen.o devtools/gossipwith.o common/cryptomsg.o common/cryptomsg.o common/crypto_sync.o $(DEVTOOLS_OBJS) $(DEVTOOLS_TOOL_OBJS): wire/wire.h diff --git a/devtools/create-gossipstore.c b/devtools/create-gossipstore.c index a5699ad9908b..42c9db3958e1 100644 --- a/devtools/create-gossipstore.c +++ b/devtools/create-gossipstore.c @@ -16,7 +16,7 @@ #include #include #include -#include +#include struct scidsat { diff --git a/devtools/decodemsg.c b/devtools/decodemsg.c index 3bfd7d6826f8..9e0650981c1c 100644 --- a/devtools/decodemsg.c +++ b/devtools/decodemsg.c @@ -5,8 +5,13 @@ #include #include #include +#if EXPERIMENTAL_FEATURES +#include +#include +#else #include #include +#endif int main(int argc, char *argv[]) { diff --git a/devtools/dump-gossipstore.c b/devtools/dump-gossipstore.c index 9d05be4dd6a4..43759e4e78da 100644 --- a/devtools/dump-gossipstore.c +++ b/devtools/dump-gossipstore.c @@ -12,7 +12,7 @@ #include #include #include -#include +#include int main(int argc, char *argv[]) { diff --git a/devtools/mkgossip.c b/devtools/mkgossip.c index 47fd81d70180..9420e79f7ae7 100644 --- a/devtools/mkgossip.c +++ b/devtools/mkgossip.c @@ -21,7 +21,7 @@ #include #include #include -#include +#include #include static bool verbose = false; diff --git a/devtools/mkquery.c b/devtools/mkquery.c index b2f4911d59da..27ca543f241c 100644 --- a/devtools/mkquery.c +++ b/devtools/mkquery.c @@ -11,7 +11,7 @@ #include #include #include -#include +#include #include static void usage(void) diff --git a/devtools/print_wire.h b/devtools/print_wire.h index a7300119cf99..58c013f5b9d1 100644 --- a/devtools/print_wire.h +++ b/devtools/print_wire.h @@ -3,8 +3,8 @@ #include #include #include -#include -#include +#include +#include struct tlv_print_record_type { u64 type; diff --git a/gossipd/Makefile b/gossipd/Makefile index 72c97eb7fba5..8deed147fdbe 100644 --- a/gossipd/Makefile +++ b/gossipd/Makefile @@ -66,7 +66,7 @@ GOSSIPD_COMMON_OBJS := \ common/wire_error.o \ connectd/connectd_gossipd_wiregen.o \ lightningd/gossip_msg.o \ - wire/onion_wiregen.o + wire/onion$(EXP)_wiregen.o lightningd/lightning_gossipd: $(GOSSIPD_OBJS) $(GOSSIPD_COMMON_OBJS) $(BITCOIN_OBJS) $(WIRE_OBJS) $(HSMD_CLIENT_OBJS) diff --git a/gossipd/gossip_generation.c b/gossipd/gossip_generation.c index c4cea5c89140..555a6150cc46 100644 --- a/gossipd/gossip_generation.c +++ b/gossipd/gossip_generation.c @@ -14,7 +14,7 @@ #include #include #include -#include +#include #include /* Create a node_announcement with the given signature. It may be NULL in the diff --git a/gossipd/gossip_store.c b/gossipd/gossip_store.c index 04396e7d3343..ed085c3805de 100644 --- a/gossipd/gossip_store.c +++ b/gossipd/gossip_store.c @@ -18,7 +18,7 @@ #include #include #include -#include +#include #include #define GOSSIP_STORE_FILENAME "gossip_store" diff --git a/gossipd/gossipd.c b/gossipd/gossipd.c index 0a6932b9d321..7e48e3acfbd4 100644 --- a/gossipd/gossipd.c +++ b/gossipd/gossipd.c @@ -63,7 +63,7 @@ #include #include #include -#include +#include #include #include diff --git a/gossipd/gossipd_wire.csv b/gossipd/gossipd_wire.csv index b91aaf9a7802..4433d104e4ce 100644 --- a/gossipd/gossipd_wire.csv +++ b/gossipd/gossipd_wire.csv @@ -1,7 +1,7 @@ #include #include #include -#include +#include # Initialize the gossip daemon. msgtype,gossipd_init,3000 diff --git a/gossipd/gossipd_wiregen.c b/gossipd/gossipd_wiregen.c index cf7992ea4a81..7f88f336f475 100644 --- a/gossipd/gossipd_wiregen.c +++ b/gossipd/gossipd_wiregen.c @@ -858,4 +858,4 @@ bool fromwire_gossipd_new_blockheight(const void *p, u32 *blockheight) return cursor != NULL; } -// SHA256STAMP:4aa62af856e3554603b0038eb0c9c6f5623cdb8d17abb6650338d1b728babd4b +// SHA256STAMP:9ac18b60ff39a11d7871d17a660aa9661a49d2585abe25d0ba0e048c32ac02eb diff --git a/gossipd/gossipd_wiregen.h b/gossipd/gossipd_wiregen.h index 5361aa77bcf8..83743247dd4a 100644 --- a/gossipd/gossipd_wiregen.h +++ b/gossipd/gossipd_wiregen.h @@ -10,7 +10,7 @@ #include #include #include -#include +#include #include #include @@ -200,4 +200,4 @@ bool fromwire_gossipd_new_blockheight(const void *p, u32 *blockheight); #endif /* LIGHTNING_GOSSIPD_GOSSIPD_WIREGEN_H */ -// SHA256STAMP:4aa62af856e3554603b0038eb0c9c6f5623cdb8d17abb6650338d1b728babd4b +// SHA256STAMP:9ac18b60ff39a11d7871d17a660aa9661a49d2585abe25d0ba0e048c32ac02eb diff --git a/gossipd/queries.c b/gossipd/queries.c index 62e5570669d8..64e4ab00e4f6 100644 --- a/gossipd/queries.c +++ b/gossipd/queries.c @@ -15,7 +15,7 @@ #include #include #include -#include +#include #include #include diff --git a/gossipd/routing.c b/gossipd/routing.c index 87f25e9546ab..ff8c15d3530b 100644 --- a/gossipd/routing.c +++ b/gossipd/routing.c @@ -21,7 +21,7 @@ #include #include #include -#include +#include #ifndef SUPERVERBOSE #define SUPERVERBOSE(...) diff --git a/gossipd/routing.h b/gossipd/routing.h index 6c0c1b53fd6e..346a5995a207 100644 --- a/gossipd/routing.h +++ b/gossipd/routing.h @@ -11,7 +11,7 @@ #include #include #include -#include +#include #include struct daemon; diff --git a/gossipd/seeker.c b/gossipd/seeker.c index 35656a2a1dec..79dda1dc7ec5 100644 --- a/gossipd/seeker.c +++ b/gossipd/seeker.c @@ -17,7 +17,7 @@ #include #include #include -#include +#include #define GOSSIP_SEEKER_INTERVAL(seeker) \ DEV_FAST_GOSSIP((seeker)->daemon->rstate->dev_fast_gossip, 5, 60) diff --git a/gossipd/test/Makefile b/gossipd/test/Makefile index 24d22f2cad43..2deca9f07330 100644 --- a/gossipd/test/Makefile +++ b/gossipd/test/Makefile @@ -17,7 +17,7 @@ GOSSIPD_TEST_COMMON_OBJS := \ common/pseudorand.o \ common/type_to_string.o \ common/utils.o \ - wire/peer_wiregen.o \ + wire/peer$(EXP)_wiregen.o \ wire/fromwire.o \ wire/towire.o diff --git a/hsmd/hsmd.c b/hsmd/hsmd.c index 63604b89b345..f4bd92a903a8 100644 --- a/hsmd/hsmd.c +++ b/hsmd/hsmd.c @@ -51,7 +51,7 @@ #include #include #include -#include +#include #include /*~ Each subdaemon is started with stdin connected to lightningd (for status diff --git a/lightningd/connect_control.c b/lightningd/connect_control.c index e2cdd3c560e6..bb5a57b2ada0 100644 --- a/lightningd/connect_control.c +++ b/lightningd/connect_control.c @@ -27,7 +27,7 @@ #include #include #include -#include +#include #include struct connect { diff --git a/lightningd/gossip_control.c b/lightningd/gossip_control.c index 071e8f67b96d..a3b608ff5647 100644 --- a/lightningd/gossip_control.c +++ b/lightningd/gossip_control.c @@ -33,7 +33,7 @@ #include #include #include -#include +#include #include static void got_txout(struct bitcoind *bitcoind, diff --git a/lightningd/htlc_end.h b/lightningd/htlc_end.h index 554350c7c4e1..c502ce44a479 100644 --- a/lightningd/htlc_end.h +++ b/lightningd/htlc_end.h @@ -7,7 +7,7 @@ #include #include #include -#include +#include /* We look up HTLCs by channel & id */ struct htlc_key { diff --git a/lightningd/htlc_set.h b/lightningd/htlc_set.h index 3657df19221c..573a92d07083 100644 --- a/lightningd/htlc_set.h +++ b/lightningd/htlc_set.h @@ -7,7 +7,7 @@ #include #include #include -#include +#include struct htlc_in; struct lightningd; diff --git a/lightningd/invoice.h b/lightningd/invoice.h index 1ccffda4a03c..4fba860649e5 100644 --- a/lightningd/invoice.h +++ b/lightningd/invoice.h @@ -1,7 +1,7 @@ #ifndef LIGHTNING_LIGHTNINGD_INVOICE_H #define LIGHTNING_LIGHTNINGD_INVOICE_H #include "config.h" -#include +#include struct amount_msat; struct htlc_set; diff --git a/lightningd/notification.h b/lightningd/notification.h index a0b248e1f368..94047988708e 100644 --- a/lightningd/notification.h +++ b/lightningd/notification.h @@ -17,7 +17,7 @@ #include #include #include -#include +#include struct onionreply; diff --git a/lightningd/peer_control.c b/lightningd/peer_control.c index f956c8d7f2c8..694447229a91 100644 --- a/lightningd/peer_control.c +++ b/lightningd/peer_control.c @@ -58,7 +58,7 @@ #include #include #include -#include +#include #include struct close_command { diff --git a/lightningd/peer_htlcs.c b/lightningd/peer_htlcs.c index 4e737bbd43f4..5729b91729bc 100644 --- a/lightningd/peer_htlcs.c +++ b/lightningd/peer_htlcs.c @@ -36,7 +36,7 @@ #include #include #include -#include +#include #include #ifndef SUPERVERBOSE diff --git a/openingd/openingd.c b/openingd/openingd.c index af5383404245..3520e3b939a9 100644 --- a/openingd/openingd.c +++ b/openingd/openingd.c @@ -52,7 +52,6 @@ #include #include #include -#include #include #include diff --git a/plugins/Makefile b/plugins/Makefile index 27160ef77b02..b8512c5cf06f 100644 --- a/plugins/Makefile +++ b/plugins/Makefile @@ -82,7 +82,7 @@ PLUGIN_COMMON_OBJS := \ common/version.o \ common/wireaddr.o \ wire/fromwire.o \ - wire/onion_wiregen.o \ + wire/onion$(EXP)_wiregen.o \ wire/tlvstream.o \ wire/towire.o @@ -95,7 +95,7 @@ plugins/fundchannel: common/addr.o $(PLUGIN_FUNDCHANNEL_OBJS) $(PLUGIN_LIB_OBJS) plugins/bcli: bitcoin/chainparams.o $(PLUGIN_BCLI_OBJS) $(PLUGIN_LIB_OBJS) $(PLUGIN_COMMON_OBJS) $(JSMN_OBJS) $(CCAN_OBJS) -plugins/keysend: bitcoin/chainparams.o wire/tlvstream.o wire/onion_wiregen.o $(PLUGIN_KEYSEND_OBJS) $(PLUGIN_LIB_OBJS) $(PLUGIN_PAY_LIB_OBJS) $(PLUGIN_COMMON_OBJS) $(JSMN_OBJS) $(CCAN_OBJS) +plugins/keysend: bitcoin/chainparams.o wire/tlvstream.o wire/onion$(EXP)_wiregen.o $(PLUGIN_KEYSEND_OBJS) $(PLUGIN_LIB_OBJS) $(PLUGIN_PAY_LIB_OBJS) $(PLUGIN_COMMON_OBJS) $(JSMN_OBJS) $(CCAN_OBJS) $(PLUGIN_KEYSEND_OBJS): $(PLUGIN_PAY_LIB_HEADER) $(PLUGIN_ALL_OBJS): $(PLUGIN_LIB_HEADER) diff --git a/plugins/keysend.c b/plugins/keysend.c index f21f51c32e53..40510ecc2d5e 100644 --- a/plugins/keysend.c +++ b/plugins/keysend.c @@ -3,7 +3,7 @@ #include #include #include -#include +#include #define PREIMAGE_TLV_TYPE 5482373484 #define KEYSEND_FEATUREBIT 55 diff --git a/plugins/libplugin-pay.h b/plugins/libplugin-pay.h index 8c5b5dfe01d7..4fc270dd9636 100644 --- a/plugins/libplugin-pay.h +++ b/plugins/libplugin-pay.h @@ -4,7 +4,7 @@ #include #include -#include +#include struct legacy_payload { struct short_channel_id scid; diff --git a/tools/test/Makefile b/tools/test/Makefile index 66cdcc816db0..73610102a45f 100644 --- a/tools/test/Makefile +++ b/tools/test/Makefile @@ -42,7 +42,7 @@ tools/test/gen_test.c.tmp.c: $(TOOLS_WIRE_DEPS) tools/test/gen_test.c: tools/test/gen_test.c.tmp.c $(EXTERNAL_HEADERS) tools/test/gen_test.h @MAKE=$(MAKE) tools/update-mocks.sh "$<" $(SUPPRESS_OUTPUT) && mv "$<" "$@" -tools/test/gen_print.h: wire/onion_wiregen.h $(TOOLS_WIRE_DEPS) +tools/test/gen_print.h: wire/onion$(EXP)_wiregen.h $(TOOLS_WIRE_DEPS) $(BOLT_GEN) -P --page header $@ test_type < tools/test/test_cases > $@ tools/test/gen_print.c: $(TOOLS_WIRE_DEPS) diff --git a/wallet/wallet.h b/wallet/wallet.h index 18f38e79e342..9c9566f8c7c0 100644 --- a/wallet/wallet.h +++ b/wallet/wallet.h @@ -20,7 +20,7 @@ #include #include #include -#include +#include struct amount_msat; struct invoices; diff --git a/wire/Makefile b/wire/Makefile index c0348e2012a1..e1ee3c937762 100644 --- a/wire/Makefile +++ b/wire/Makefile @@ -2,15 +2,16 @@ WIRE_HEADERS := wire/onion_defs.h \ wire/peer_wire.h \ + wire/onion_wire.h \ wire/tlvstream.h \ wire/wire.h \ wire/wire_sync.h \ wire/wire_io.h \ - wire/peer_wiregen.h \ - wire/onion_wiregen.h \ + wire/peer$(EXP)_wiregen.h \ + wire/onion$(EXP)_wiregen.h \ wire/common_wiregen.h \ - wire/peer_printgen.h \ - wire/onion_printgen.h + wire/peer$(EXP)_printgen.h \ + wire/onion$(EXP)_printgen.h # We don't include peer_printgen/onion_printgen here since most don't need it. WIRE_SRC := wire/wire_sync.c \ @@ -20,12 +21,12 @@ WIRE_SRC := wire/wire_sync.c \ wire/tlvstream.c \ wire/towire.c \ wire/common_wiregen.c \ - wire/peer_wiregen.c \ - wire/onion_wiregen.c + wire/peer$(EXP)_wiregen.c \ + wire/onion$(EXP)_wiregen.c WIRE_PRINT_SRC := \ - wire/onion_printgen.c \ - wire/peer_printgen.c + wire/onion$(EXP)_printgen.c \ + wire/peer$(EXP)_printgen.c WIRE_OBJS := $(WIRE_SRC:.c=.o) WIRE_PRINT_OBJS := $(WIRE_PRINT_SRC:.c=.o) @@ -61,23 +62,13 @@ ifeq ($(EXPERIMENTAL_FEATURES),1) EXPERIMENTAL_PEER_PATCHES := $(sort $(wildcard wire/extracted_peer_experimental_*)) EXPERIMENTAL_ONION_PATCHES := $(sort $(wildcard wire/extracted_onion_experimental_*)) -wire/peer_wire.csv: wire/extracted_peer_wire_csv $(EXPERIMENTAL_PEER_PATCHES) +wire/peer_exp_wire.csv: wire/peer_wire_csv $(EXPERIMENTAL_PEER_PATCHES) @set -e; trap "rm -f $@.$$$$" 0; cp $< $@.$$$$; for exp in $(EXPERIMENTAL_PEER_PATCHES); do patch $@.$$$$ $$exp >/dev/null ; done; mv $@.$$$$ $@ -wire/onion_wire.csv: wire/extracted_onion_wire_csv $(EXPERIMENTAL_ONION_PATCHES) +wire/onion_exp_wire.csv: wire/onion_wire_csv $(EXPERIMENTAL_ONION_PATCHES) @set -e; trap "rm -f $@.$$$$" 0; cp $< $@.$$$$; for exp in $(EXPERIMENTAL_ONION_PATCHES); do patch $@.$$$$ $$exp; done >/dev/null ; mv $@.$$$$ $@ - -else # /* EXPERIMENTAL_FEATURES */ -wire/peer_wire.csv: wire/extracted_peer_wire_csv - @cp $< $@ - -wire/onion_wire.csv: wire/extracted_onion_wire_csv - @cp $< $@ endif -# We (may) need to rebuild these if config changes -wire/peer_wire.csv wire/onion_wire.csv: config.vars - # tlvs_n1 and n2 are used for test vectors, thus not referenced: expose them # for testing and to prevent compile error about them being unused. # This will be easier if test vectors are moved to separate files. @@ -90,6 +81,12 @@ wire/onion_wiregen.h_args := --include='bitcoin/short_channel_id.h' --include='b wire/onion_wiregen.c_args := -s --expose-tlv-type=tlv_payload +# Same for _exp versions +wire/peer_exp_wiregen.h_args := $(wire/peer_wiregen.h_args) +wire/peer_exp_wiregen.c_args := $(wire/peer_wiregen.c_args) +wire/onion_exp_wiregen.h_args := $(wire/onion_wiregen.h_args) +wire/onion_exp_wiregen.c_args := $(wire/onion_wiregen.c_args) + maintainer-clean: wire-maintainer-clean wire-maintainer-clean: diff --git a/wire/onion_wire.h b/wire/onion_wire.h new file mode 100644 index 000000000000..63bea468409c --- /dev/null +++ b/wire/onion_wire.h @@ -0,0 +1,12 @@ +#ifndef LIGHTNING_WIRE_ONION_WIRE_H +#define LIGHTNING_WIRE_ONION_WIRE_H +#include "config.h" +#include + +#if EXPERIMENTAL_FEATURES +#include +#else +#include +#endif + +#endif /* LIGHTNING_WIRE_ONION_WIRE_H */ diff --git a/wire/extracted_onion_wire_csv b/wire/onion_wire_csv similarity index 100% rename from wire/extracted_onion_wire_csv rename to wire/onion_wire_csv diff --git a/wire/peer_wire.h b/wire/peer_wire.h index 008410304123..12c951b8ff3e 100644 --- a/wire/peer_wire.h +++ b/wire/peer_wire.h @@ -2,7 +2,12 @@ #define LIGHTNING_WIRE_PEER_WIRE_H #include "config.h" #include + +#if EXPERIMENTAL_FEATURES +#include +#else #include +#endif /* BOLT #1: * diff --git a/wire/extracted_peer_wire_csv b/wire/peer_wire_csv similarity index 100% rename from wire/extracted_peer_wire_csv rename to wire/peer_wire_csv diff --git a/wire/test/Makefile b/wire/test/Makefile index db7bf7a54d37..154f92ea3e62 100644 --- a/wire/test/Makefile +++ b/wire/test/Makefile @@ -21,4 +21,4 @@ $(WIRE_TEST_OBJS): $(WIRE_HEADERS) $(WIRE_SRC) $(WIRE_PRINT_SRC) wire-tests: $(WIRE_TEST_PROGRAMS:%=unittest/%) -wire/test/run-peer-wire: wire/peer_wiregen.o common/bigsize.o +wire/test/run-peer-wire: wire/peer$(EXP)_wiregen.o common/bigsize.o diff --git a/wire/test/run-peer-wire.c b/wire/test/run-peer-wire.c index c5b7e54f3584..a9acefb931cd 100644 --- a/wire/test/run-peer-wire.c +++ b/wire/test/run-peer-wire.c @@ -14,7 +14,7 @@ #include #include #include -#include +#include extern secp256k1_context *secp256k1_ctx; diff --git a/wire/test/run-tlvstream.c b/wire/test/run-tlvstream.c index dd05357c590c..f7991edb67e8 100644 --- a/wire/test/run-tlvstream.c +++ b/wire/test/run-tlvstream.c @@ -13,7 +13,11 @@ static const char *reason; #include #include +#if EXPERIMENTAL_FEATURES +#include +#else #include +#endif #include #include From 96d260b787032169fca4247fb3bad3cf9596db54 Mon Sep 17 00:00:00 2001 From: Rusty Russell Date: Fri, 28 Aug 2020 12:00:57 +0930 Subject: [PATCH 05/30] wallet: fix reference to potentially-undefined variable. If all is set, diff may not be: ==1230518== Use of uninitialised value of size 8 ==1230518== at 0x4C5781B: _itoa_word (_itoa.c:179) ==1230518== by 0x4C736F4: __vfprintf_internal (vfprintf-internal.c:1687) ==1230518== by 0x4C88119: __vsnprintf_internal (vsnprintf.c:114) ==1230518== by 0x1D44B6: do_vfmt (str.c:66) ==1230518== by 0x1D45A0: tal_vfmt_ (str.c:92) ==1230518== by 0x1D4401: tal_fmt_ (str.c:44) ==1230518== by 0x15D30F: fmt_amount_sat (amount.c:60) ==1230518== by 0x15D338: fmt_amount_sat_ (amount.c:62) ==1230518== by 0x178C45: type_to_string_ (type_to_string.c:35) ==1230518== by 0x1B8F75: json_fundpsbt (reservation.c:394) ==1230518== by 0x12D0EC: command_exec (jsonrpc.c:602) ==1230518== by 0x12D598: rpc_command_hook_callback (jsonrpc.c:712) Signed-off-by: Rusty Russell --- wallet/reservation.c | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/wallet/reservation.c b/wallet/reservation.c index 5b3a764b510f..2e0d02c957cf 100644 --- a/wallet/reservation.c +++ b/wallet/reservation.c @@ -324,9 +324,10 @@ static struct command_result *json_fundpsbt(struct command *cmd, struct amount_sat, amount), tal_count(utxos), - type_to_string(tmpctx, - struct amount_sat, - &diff)); + all ? "all" + : type_to_string(tmpctx, + struct amount_sat, + &diff)); } if (all) { From 8d5ff210bf592db80b6ffa37c9fd551ffbb4e180 Mon Sep 17 00:00:00 2001 From: Rusty Russell Date: Fri, 28 Aug 2020 12:01:57 +0930 Subject: [PATCH 06/30] txprepare: a plugin to replace `txprepare` (not yet active). This uses `fundpsbt` and similar to simulate the txprepare command. It has one difference (when complete), in that it those reservations are now timed and don't get reset on restart. It also doesn't have the restriction that `all` can only be used with no other output, as I didn't realize that when I implemented it! Note that change is now inserted in a random position, not sorted into BIP69 order. Signed-off-by: Rusty Russell --- plugins/Makefile | 9 +- plugins/txprepare.c | 305 ++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 313 insertions(+), 1 deletion(-) create mode 100644 plugins/txprepare.c diff --git a/plugins/Makefile b/plugins/Makefile index b8512c5cf06f..b59e6119a517 100644 --- a/plugins/Makefile +++ b/plugins/Makefile @@ -7,6 +7,9 @@ PLUGIN_AUTOCLEAN_OBJS := $(PLUGIN_AUTOCLEAN_SRC:.c=.o) PLUGIN_FUNDCHANNEL_SRC := plugins/fundchannel.c PLUGIN_FUNDCHANNEL_OBJS := $(PLUGIN_FUNDCHANNEL_SRC:.c=.o) +PLUGIN_TXPREPARE_SRC := plugins/txprepare.c +PLUGIN_TXPREPARE_OBJS := $(PLUGIN_TXPREPARE_SRC:.c=.o) + PLUGIN_BCLI_SRC := plugins/bcli.c PLUGIN_BCLI_OBJS := $(PLUGIN_BCLI_SRC:.c=.o) @@ -26,6 +29,7 @@ PLUGIN_ALL_SRC := \ $(PLUGIN_BCLI_SRC) \ $(PLUGIN_FUNDCHANNEL_SRC) \ $(PLUGIN_KEYSEND_SRC) \ + $(PLUGIN_TXPREPARE_SRC) \ $(PLUGIN_LIB_SRC) \ $(PLUGIN_PAY_LIB_SRC) \ $(PLUGIN_PAY_SRC) @@ -39,7 +43,8 @@ PLUGINS := \ plugins/bcli \ plugins/fundchannel \ plugins/keysend \ - plugins/pay + plugins/pay \ + plugins/txprepare # Make sure these depend on everything. ALL_C_SOURCES += $(PLUGIN_ALL_SRC) @@ -93,6 +98,8 @@ plugins/autoclean: bitcoin/chainparams.o $(PLUGIN_AUTOCLEAN_OBJS) $(PLUGIN_LIB_O plugins/fundchannel: common/addr.o $(PLUGIN_FUNDCHANNEL_OBJS) $(PLUGIN_LIB_OBJS) $(PLUGIN_COMMON_OBJS) $(JSMN_OBJS) $(CCAN_OBJS) +plugins/txprepare: bitcoin/chainparams.o $(PLUGIN_TXPREPARE_OBJS) $(PLUGIN_LIB_OBJS) $(PLUGIN_COMMON_OBJS) $(JSMN_OBJS) $(CCAN_OBJS) + plugins/bcli: bitcoin/chainparams.o $(PLUGIN_BCLI_OBJS) $(PLUGIN_LIB_OBJS) $(PLUGIN_COMMON_OBJS) $(JSMN_OBJS) $(CCAN_OBJS) plugins/keysend: bitcoin/chainparams.o wire/tlvstream.o wire/onion$(EXP)_wiregen.o $(PLUGIN_KEYSEND_OBJS) $(PLUGIN_LIB_OBJS) $(PLUGIN_PAY_LIB_OBJS) $(PLUGIN_COMMON_OBJS) $(JSMN_OBJS) $(CCAN_OBJS) diff --git a/plugins/txprepare.c b/plugins/txprepare.c new file mode 100644 index 000000000000..96977fdf3693 --- /dev/null +++ b/plugins/txprepare.c @@ -0,0 +1,305 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +struct tx_output { + struct amount_sat amount; + const u8 *script; +}; + +struct txprepare { + struct tx_output *outputs; + struct amount_sat output_total; + /* Weight for core + outputs */ + size_t weight; + + /* Which output is 'all', or -1 (not counted in output_total!) */ + int all_output_idx; + + /* Once we have a PSBT, it goes here. */ + struct wally_psbt *psbt; + u32 feerate; + + /* Once we have reserved all the inputs, this is set. */ + struct amount_sat change_amount; +}; + + +static struct wally_psbt *json_tok_psbt(const tal_t *ctx, + const char *buffer, + const jsmntok_t *tok) +{ + return psbt_from_b64(ctx, buffer + tok->start, tok->end - tok->start); +} + +static struct command_result *param_outputs(struct command *cmd, + const char *name, + const char *buffer, + const jsmntok_t *tok, + struct txprepare *txp) +{ + size_t i; + const jsmntok_t *t; + + txp->outputs = tal_arr(txp, struct tx_output, tok->size); + txp->output_total = AMOUNT_SAT(0); + txp->all_output_idx = -1; + + /* We assume < 253 inputs, and if we're wrong, the fee + * difference is trivial. */ + txp->weight = bitcoin_tx_core_weight(1, tal_count(txp->outputs)); + + json_for_each_arr(i, t, tok) { + enum address_parse_result res; + struct tx_output *out = &txp->outputs[i]; + + /* output format: {destination: amount} */ + if (t->type != JSMN_OBJECT) + return command_fail(cmd, JSONRPC2_INVALID_PARAMS, + "The output format must be " + "{destination: amount}"); + res = json_to_address_scriptpubkey(cmd, + chainparams, + buffer, &t[1], + &out->script); + if (res == ADDRESS_PARSE_UNRECOGNIZED) + return command_fail(cmd, JSONRPC2_INVALID_PARAMS, + "Could not parse destination address"); + else if (res == ADDRESS_PARSE_WRONG_NETWORK) + return command_fail(cmd, JSONRPC2_INVALID_PARAMS, + "Destination address is not on network %s", + chainparams->network_name); + + if (!json_to_sat_or_all(buffer, &t[2], &out->amount)) + return command_fail(cmd, JSONRPC2_INVALID_PARAMS, + "'%.*s' is a invalid satoshi amount", + t[2].end - t[2].start, + buffer + t[2].start); + + if (amount_sat_eq(out->amount, AMOUNT_SAT(-1ULL))) { + if (txp->all_output_idx != -1) + return command_fail(cmd, JSONRPC2_INVALID_PARAMS, + "Cannot use 'all' in" + " two outputs"); + txp->all_output_idx = i; + } else { + if (!amount_sat_add(&txp->output_total, + txp->output_total, + out->amount)) + return command_fail(cmd, JSONRPC2_INVALID_PARAMS, + "Output amount overflow"); + } + txp->weight += bitcoin_tx_output_weight(tal_bytelen(out->script)); + } + return NULL; +} + +static struct command_result *finish_txprepare(struct command *cmd, + struct txprepare *txp) +{ + struct json_stream *out; + struct wally_tx *tx; + struct bitcoin_txid txid; + + /* Add the outputs they gave us */ + for (size_t i = 0; i < tal_count(txp->outputs); i++) { + struct wally_tx_output *out; + + out = wally_tx_output(txp->outputs[i].script, + txp->outputs[i].amount); + if (!out) + return command_fail(cmd, JSONRPC2_INVALID_PARAMS, + "Invalid output %zi (%s:%s)", i, + tal_hex(tmpctx, + txp->outputs[i].script), + type_to_string(tmpctx, + struct amount_sat, + &txp->outputs[i].amount)); + psbt_add_output(txp->psbt, out, i); + } + + psbt_txid(txp->psbt, &txid, &tx); + out = jsonrpc_stream_success(cmd); + json_add_hex_talarr(out, "unsigned_tx", linearize_wtx(tmpctx, tx)); + json_add_txid(out, "txid", &txid); + return command_finished(cmd, out); +} + +/* newaddr has given us a change address. */ +static struct command_result *newaddr_done(struct command *cmd, + const char *buf, + const jsmntok_t *result, + struct txprepare *txp) +{ + size_t num = tal_count(txp->outputs), pos; + const jsmntok_t *addr = json_get_member(buf, result, "bech32"); + + /* Insert change in random position in outputs */ + tal_resize(&txp->outputs, num+1); + pos = pseudorand(num+1); + memmove(txp->outputs + pos + 1, + txp->outputs + pos, + sizeof(txp->outputs[0]) * (num - pos)); + + txp->outputs[pos].amount = txp->change_amount; + if (json_to_address_scriptpubkey(txp, chainparams, buf, addr, + &txp->outputs[pos].script) + != ADDRESS_PARSE_SUCCESS) { + return command_fail(cmd, LIGHTNINGD, + "Change address '%.*s' unparsable?", + addr->end - addr->start, + buf + addr->start); + } + + return finish_txprepare(cmd, txp); +} + +static bool resolve_all_output_amount(struct txprepare *txp, + struct amount_sat excess) +{ + if (!amount_sat_greater_eq(excess, chainparams->dust_limit)) + return false; + + assert(amount_sat_eq(txp->outputs[txp->all_output_idx].amount, + AMOUNT_SAT(-1ULL))); + txp->outputs[txp->all_output_idx].amount = excess; + return true; +} + +/* fundpsbt gets a viable PSBT for us. */ +static struct command_result *fundpsbt_done(struct command *cmd, + const char *buf, + const jsmntok_t *result, + struct txprepare *txp) +{ + const jsmntok_t *psbttok; + struct out_req *req; + struct amount_sat excess; + + psbttok = json_get_member(buf, result, "psbt"); + txp->psbt = json_tok_psbt(txp, buf, psbttok); + if (!txp->psbt) + return command_fail(cmd, LIGHTNINGD, + "Unparsable psbt: '%.*s'", + psbttok->end - psbttok->start, + buf + psbttok->start); + + if (!json_to_number(buf, json_get_member(buf, result, "feerate_per_kw"), + &txp->feerate)) + return command_fail(cmd, LIGHTNINGD, + "Unparsable feerate_per_kw: '%.*s'", + result->end - result->start, + buf + result->start); + + if (!json_to_sat(buf, json_get_member(buf, result, "excess_msat"), + &excess)) + return command_fail(cmd, LIGHTNINGD, + "Unparsable excess_msat: '%.*s'", + result->end - result->start, + buf + result->start); + + /* If we have an "all" output, now we can derive its value: excess + * in this case will be total value after inputs paid for themselves. */ + if (txp->all_output_idx != -1) { + if (!resolve_all_output_amount(txp, excess)) + return command_fail(cmd, FUND_CANNOT_AFFORD, + "Insufficient funds to make" + " 'all' output"); + + /* Never produce change if they asked for all */ + excess = AMOUNT_SAT(0); + } + + /* So, do we need change? */ + txp->change_amount = change_amount(excess, txp->feerate); + if (amount_sat_eq(txp->change_amount, AMOUNT_SAT(0))) + return finish_txprepare(cmd, txp); + + /* Ask for a change address */ + req = jsonrpc_request_start(cmd->plugin, cmd, + "newaddr", + newaddr_done, + /* It would be nice to unreserve inputs, + * but probably won't happen. */ + forward_error, + txp); + return send_outreq(cmd->plugin, req); +} + +static struct command_result *json_txprepare(struct command *cmd, + const char *buffer, + const jsmntok_t *params) +{ + struct txprepare *txp = tal(cmd, struct txprepare); + struct out_req *req; + const char *feerate; + unsigned int *minconf; + const jsmntok_t *utxos; + + if (!param(cmd, buffer, params, + p_req("outputs", param_outputs, txp), + p_opt("feerate", param_string, &feerate), + p_opt_def("minconf", param_number, &minconf, 1), + p_opt("utxos", param_tok, &utxos), + NULL)) + return command_param_failed(); + + /* p_opt_def doesn't compile with strings... */ + if (!feerate) + feerate = "opening"; + + if (utxos) { + return command_done_err(cmd, + JSONRPC2_INVALID_PARAMS, + "FIXME: utxos not implemented!", + NULL); + } + + /* Otherwise, we can now try to gather UTXOs. */ + req = jsonrpc_request_start(cmd->plugin, cmd, "fundpsbt", + fundpsbt_done, forward_error, + txp); + + if (txp->all_output_idx == -1) + json_add_amount_sat_only(req->js, "satoshi", txp->output_total); + else + json_add_string(req->js, "satoshi", "all"); + + json_add_u32(req->js, "startweight", txp->weight); + + /* Pass through feerate and minconf */ + json_add_string(req->js, "feerate", feerate); + json_add_u32(req->js, "minconf", *minconf); + return send_outreq(cmd->plugin, req); +} + +static const struct plugin_command commands[] = { + { + "newtxprepare", + "bitcoin", + "Create a transaction, with option to spend in future (either txsend and txdiscard)", + "Create an unsigned transaction paying {outputs} with optional {feerate}, {minconf} and {utxos}", + json_txprepare + }, +}; + +int main(int argc, char *argv[]) +{ + setup_locale(); + plugin_main(argv, NULL, PLUGIN_RESTARTABLE, true, NULL, commands, + ARRAY_SIZE(commands), NULL, 0, NULL, 0, NULL); +} From 74c6307c5430dc41f6310e884f7d42bdd872bd58 Mon Sep 17 00:00:00 2001 From: Rusty Russell Date: Fri, 28 Aug 2020 12:02:57 +0930 Subject: [PATCH 07/30] plugins/txprepare: use utxopsbt to handle utxos arg. Very similar to fundpsbt, but takes 'utxos' instead of 'minconf'. Signed-off-by: Rusty Russell --- plugins/txprepare.c | 37 ++++++++++++++++++------------------- 1 file changed, 18 insertions(+), 19 deletions(-) diff --git a/plugins/txprepare.c b/plugins/txprepare.c index 96977fdf3693..76bbedc109a5 100644 --- a/plugins/txprepare.c +++ b/plugins/txprepare.c @@ -180,11 +180,11 @@ static bool resolve_all_output_amount(struct txprepare *txp, return true; } -/* fundpsbt gets a viable PSBT for us. */ -static struct command_result *fundpsbt_done(struct command *cmd, - const char *buf, - const jsmntok_t *result, - struct txprepare *txp) +/* fundpsbt/utxopsbt gets a viable PSBT for us. */ +static struct command_result *psbt_created(struct command *cmd, + const char *buf, + const jsmntok_t *result, + struct txprepare *txp) { const jsmntok_t *psbttok; struct out_req *req; @@ -246,15 +246,14 @@ static struct command_result *json_txprepare(struct command *cmd, { struct txprepare *txp = tal(cmd, struct txprepare); struct out_req *req; - const char *feerate; + const char *feerate, *utxos; unsigned int *minconf; - const jsmntok_t *utxos; if (!param(cmd, buffer, params, p_req("outputs", param_outputs, txp), p_opt("feerate", param_string, &feerate), p_opt_def("minconf", param_number, &minconf, 1), - p_opt("utxos", param_tok, &utxos), + p_opt("utxos", param_string, &utxos), NULL)) return command_param_failed(); @@ -262,18 +261,20 @@ static struct command_result *json_txprepare(struct command *cmd, if (!feerate) feerate = "opening"; + /* These calls are deliberately very similar, but utxopsbt wants utxos, + * and fundpsbt wants minconf */ if (utxos) { - return command_done_err(cmd, - JSONRPC2_INVALID_PARAMS, - "FIXME: utxos not implemented!", - NULL); + req = jsonrpc_request_start(cmd->plugin, cmd, "utxopsbt", + psbt_created, forward_error, + txp); + json_add_jsonstr(req->js, "utxos", utxos); + } else { + req = jsonrpc_request_start(cmd->plugin, cmd, "fundpsbt", + psbt_created, forward_error, + txp); + json_add_u32(req->js, "minconf", *minconf); } - /* Otherwise, we can now try to gather UTXOs. */ - req = jsonrpc_request_start(cmd->plugin, cmd, "fundpsbt", - fundpsbt_done, forward_error, - txp); - if (txp->all_output_idx == -1) json_add_amount_sat_only(req->js, "satoshi", txp->output_total); else @@ -281,9 +282,7 @@ static struct command_result *json_txprepare(struct command *cmd, json_add_u32(req->js, "startweight", txp->weight); - /* Pass through feerate and minconf */ json_add_string(req->js, "feerate", feerate); - json_add_u32(req->js, "minconf", *minconf); return send_outreq(cmd->plugin, req); } From 3f57249f085099fca3d60b6297b5b14995ffe48d Mon Sep 17 00:00:00 2001 From: Rusty Russell Date: Fri, 28 Aug 2020 12:03:57 +0930 Subject: [PATCH 08/30] reservation: return FUNDING_STILL_SYNCING_BITCOIN if we might be missing funds. This is useful, and also makes us pass the testsuite. Signed-off-by: Rusty Russell --- wallet/reservation.c | 28 ++++++++++++++++++---------- 1 file changed, 18 insertions(+), 10 deletions(-) diff --git a/wallet/reservation.c b/wallet/reservation.c index 2e0d02c957cf..7680614f03be 100644 --- a/wallet/reservation.c +++ b/wallet/reservation.c @@ -310,19 +310,22 @@ static struct command_result *json_fundpsbt(struct command *cmd, } /* If they said "all", we expect to run out of utxos. */ - if (all) { - /* If we have none at all though, fail */ - if (!tal_count(utxos)) - return command_fail(cmd, FUND_CANNOT_AFFORD, - "No available UTXOs"); + if (all && tal_count(utxos)) break; - } + + /* Since it's possible the lack of utxos is because we haven't + * finished syncing yet, report a sync timing error first */ + if (!topology_synced(cmd->ld->topology)) + return command_fail(cmd, + FUNDING_STILL_SYNCING_BITCOIN, + "Cannot afford: still syncing with bitcoin network..."); return command_fail(cmd, FUND_CANNOT_AFFORD, "Could not afford %s using all %zu available UTXOs: %s short", - type_to_string(tmpctx, - struct amount_sat, - amount), + all ? "all" + : type_to_string(tmpctx, + struct amount_sat, + amount), tal_count(utxos), all ? "all" : type_to_string(tmpctx, @@ -334,11 +337,16 @@ static struct command_result *json_fundpsbt(struct command *cmd, /* Anything above 0 is "excess" */ if (!inputs_sufficient(input, AMOUNT_SAT(0), *feerate_per_kw, *weight, - &diff)) + &diff)) { + if (!topology_synced(cmd->ld->topology)) + return command_fail(cmd, + FUNDING_STILL_SYNCING_BITCOIN, + "Cannot afford: still syncing with bitcoin network..."); return command_fail(cmd, FUND_CANNOT_AFFORD, "All %zu inputs could not afford" " fees", tal_count(utxos)); + } } return finish_psbt(cmd, utxos, *feerate_per_kw, *weight, diff, *reserve, From 522459ba6eea87d28dfd89b5ad3c47d437c21ed0 Mon Sep 17 00:00:00 2001 From: Rusty Russell Date: Fri, 28 Aug 2020 12:04:57 +0930 Subject: [PATCH 09/30] txprepare: add txdiscard functionality. Signed-off-by: Rusty Russell --- plugins/txprepare.c | 91 ++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 86 insertions(+), 5 deletions(-) diff --git a/plugins/txprepare.c b/plugins/txprepare.c index 76bbedc109a5..25440ca65db9 100644 --- a/plugins/txprepare.c +++ b/plugins/txprepare.c @@ -5,6 +5,7 @@ #include #include #include +#include #include #include #include @@ -38,6 +39,14 @@ struct txprepare { struct amount_sat change_amount; }; +struct unreleased_tx { + struct list_node list; + struct bitcoin_txid txid; + struct wally_tx *tx; + struct wally_psbt *psbt; +}; + +static LIST_HEAD(unreleased_txs); static struct wally_psbt *json_tok_psbt(const tal_t *ctx, const char *buffer, @@ -112,8 +121,7 @@ static struct command_result *finish_txprepare(struct command *cmd, struct txprepare *txp) { struct json_stream *out; - struct wally_tx *tx; - struct bitcoin_txid txid; + struct unreleased_tx *utx; /* Add the outputs they gave us */ for (size_t i = 0; i < tal_count(txp->outputs); i++) { @@ -132,10 +140,14 @@ static struct command_result *finish_txprepare(struct command *cmd, psbt_add_output(txp->psbt, out, i); } - psbt_txid(txp->psbt, &txid, &tx); + utx = tal(NULL, struct unreleased_tx); + utx->psbt = tal_steal(utx, txp->psbt); + psbt_txid(txp->psbt, &utx->txid, &utx->tx); + list_add(&unreleased_txs, &utx->list); + out = jsonrpc_stream_success(cmd); - json_add_hex_talarr(out, "unsigned_tx", linearize_wtx(tmpctx, tx)); - json_add_txid(out, "txid", &txid); + json_add_hex_talarr(out, "unsigned_tx", linearize_wtx(tmpctx, utx->tx)); + json_add_txid(out, "txid", &utx->txid); return command_finished(cmd, out); } @@ -286,6 +298,68 @@ static struct command_result *json_txprepare(struct command *cmd, return send_outreq(cmd->plugin, req); } +/* Called after we've unreserved the inputs. */ +static struct command_result *unreserve_done(struct command *cmd, + const char *buf, + const jsmntok_t *result, + struct unreleased_tx *utx) +{ + struct json_stream *out; + + out = jsonrpc_stream_success(cmd); + json_add_hex_talarr(out, "unsigned_tx", linearize_wtx(tmpctx, utx->tx)); + json_add_txid(out, "txid", &utx->txid); + + return command_finished(cmd, out); +} + +static struct command_result *param_unreleased_txid(struct command *cmd, + const char *name, + const char *buffer, + const jsmntok_t *tok, + struct unreleased_tx **utx) +{ + struct command_result *res; + struct bitcoin_txid *txid; + + res = param_txid(cmd, name, buffer, tok, &txid); + if (res) + return res; + + list_for_each(&unreleased_txs, (*utx), list) { + if (bitcoin_txid_eq(txid, &(*utx)->txid)) + return NULL; + } + + return command_fail(cmd, JSONRPC2_INVALID_PARAMS, + "not an unreleased txid '%s'", + type_to_string(tmpctx, struct bitcoin_txid, txid)); +} + +static struct command_result *json_txdiscard(struct command *cmd, + const char *buffer, + const jsmntok_t *params) +{ + struct unreleased_tx *utx; + struct out_req *req; + + if (!param(cmd, buffer, params, + p_req("txid", param_unreleased_txid, &utx), + NULL)) + return command_param_failed(); + + /* Remove from list now, to avoid races! */ + list_del_from(&unreleased_txs, &utx->list); + /* Whatever happens, we free it once this command is done. */ + tal_steal(cmd, utx); + + req = jsonrpc_request_start(cmd->plugin, cmd, "unreserveinputs", + unreserve_done, forward_error, + utx); + json_add_psbt(req->js, "psbt", utx->psbt); + return send_outreq(cmd->plugin, req); +} + static const struct plugin_command commands[] = { { "newtxprepare", @@ -294,6 +368,13 @@ static const struct plugin_command commands[] = { "Create an unsigned transaction paying {outputs} with optional {feerate}, {minconf} and {utxos}", json_txprepare }, + { + "newtxdiscard", + "bitcoin", + "Discard a transaction created by txprepare", + "Discard a transcation by {txid}", + json_txdiscard + }, }; int main(int argc, char *argv[]) From 3b1a226f67095a202d45a2e2ec61c0d56673129f Mon Sep 17 00:00:00 2001 From: Rusty Russell Date: Fri, 28 Aug 2020 12:05:57 +0930 Subject: [PATCH 10/30] txprepare: add txsend functionality. Signed-off-by: Rusty Russell --- plugins/txprepare.c | 79 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 79 insertions(+) diff --git a/plugins/txprepare.c b/plugins/txprepare.c index 25440ca65db9..57e1d4964782 100644 --- a/plugins/txprepare.c +++ b/plugins/txprepare.c @@ -360,6 +360,78 @@ static struct command_result *json_txdiscard(struct command *cmd, return send_outreq(cmd->plugin, req); } +/* Called after lightningd has broadcast the transaction. */ +static struct command_result *sendpsbt_done(struct command *cmd, + const char *buf, + const jsmntok_t *result, + struct unreleased_tx *utx) +{ + struct json_stream *out; + + out = jsonrpc_stream_success(cmd); + json_add_hex_talarr(out, "tx", linearize_wtx(tmpctx, utx->tx)); + json_add_txid(out, "txid", &utx->txid); + return command_finished(cmd, out); +} + +/* Called after lightningd has signed the inputs. */ +static struct command_result *signpsbt_done(struct command *cmd, + const char *buf, + const jsmntok_t *result, + struct unreleased_tx *utx) +{ + struct out_req *req; + const jsmntok_t *psbttok = json_get_member(buf, result, "signed_psbt"); + struct bitcoin_txid txid; + + tal_free(utx->psbt); + utx->psbt = json_tok_psbt(utx, buf, psbttok); + /* Replace with signed tx. */ + tal_free(utx->tx); + + /* The txid from the final should match our expectation. */ + psbt_txid(utx->psbt, &txid, &utx->tx); + if (!bitcoin_txid_eq(&txid, &utx->txid)) { + return command_fail(cmd, LIGHTNINGD, + "Signed tx changed txid? Had '%s' now '%s'", + tal_hex(tmpctx, + linearize_wtx(tmpctx, utx->tx)), + tal_hex(tmpctx, + linearize_wtx(tmpctx, + utx->psbt->tx))); + } + + req = jsonrpc_request_start(cmd->plugin, cmd, "sendpsbt", + sendpsbt_done, forward_error, + utx); + json_add_psbt(req->js, "psbt", utx->psbt); + return send_outreq(cmd->plugin, req); +} + +static struct command_result *json_txsend(struct command *cmd, + const char *buffer, + const jsmntok_t *params) +{ + struct unreleased_tx *utx; + struct out_req *req; + + if (!param(cmd, buffer, params, + p_req("txid", param_unreleased_txid, &utx), + NULL)) + return command_param_failed(); + + /* Remove from list now, to avoid races! */ + list_del_from(&unreleased_txs, &utx->list); + /* If things go wrong, free it. */ + tal_steal(cmd, utx); + + req = jsonrpc_request_start(cmd->plugin, cmd, "signpsbt", + signpsbt_done, forward_error, + utx); + json_add_psbt(req->js, "psbt", utx->psbt); + return send_outreq(cmd->plugin, req); +} + static const struct plugin_command commands[] = { { "newtxprepare", @@ -375,6 +447,13 @@ static const struct plugin_command commands[] = { "Discard a transcation by {txid}", json_txdiscard }, + { + "newtxsend", + "bitcoin", + "Send a transaction created by txprepare", + "Send a transacation by {txid}", + json_txsend + }, }; int main(int argc, char *argv[]) From 843d3d674593389e6242dd32bda7d8d3b3a83a71 Mon Sep 17 00:00:00 2001 From: Rusty Russell Date: Fri, 28 Aug 2020 12:06:57 +0930 Subject: [PATCH 11/30] pytest: remove test_txprepare_restart This tested that we unreserved everything on restart, which we no longer do. Signed-off-by: Rusty Russell --- tests/test_wallet.py | 100 +------------------------------------------ 1 file changed, 1 insertion(+), 99 deletions(-) diff --git a/tests/test_wallet.py b/tests/test_wallet.py index d2d0833df3aa..3415f58322c8 100644 --- a/tests/test_wallet.py +++ b/tests/test_wallet.py @@ -5,7 +5,7 @@ from flaky import flaky # noqa: F401 from pyln.client import RpcError, Millisatoshi from utils import ( - only_one, wait_for, sync_blockheight, EXPERIMENTAL_FEATURES, COMPAT, + only_one, wait_for, sync_blockheight, EXPERIMENTAL_FEATURES, VALGRIND, check_coin_moves ) @@ -232,50 +232,6 @@ def test_addfunds_from_block(node_factory, bitcoind): assert output['address'] == addr -@unittest.skipIf(not COMPAT, "needs COMPAT=1") -def test_deprecated_txprepare(node_factory, bitcoind): - """Test the deprecated old-style: - txprepare {destination} {satoshi} {feerate} {minconf} - """ - amount = 10**4 - l1 = node_factory.get_node(options={'allow-deprecated-apis': True}) - addr = l1.rpc.newaddr()['bech32'] - - for i in range(7): - l1.fundwallet(10**8) - - bitcoind.generate_block(1) - sync_blockheight(bitcoind, [l1]) - - wait_for(lambda: len(l1.rpc.listfunds()['outputs']) == 7) - - # Array type - with pytest.raises(RpcError, match=r'.* should be an amount in satoshis or all, not .*'): - l1.rpc.call('txprepare', [addr, 'slow']) - - with pytest.raises(RpcError, match=r'Need set \'satoshi\' field.'): - l1.rpc.call('txprepare', [addr]) - - with pytest.raises(RpcError, match=r'Could not parse destination address.*'): - l1.rpc.call('txprepare', [Millisatoshi(amount * 100), 'slow', 1]) - - l1.rpc.call('txprepare', [addr, Millisatoshi(amount * 100), 'slow', 1]) - l1.rpc.call('txprepare', [addr, Millisatoshi(amount * 100), 'normal']) - l1.rpc.call('txprepare', [addr, Millisatoshi(amount * 100), None, 1]) - l1.rpc.call('txprepare', [addr, Millisatoshi(amount * 100)]) - - # Object type - with pytest.raises(RpcError, match=r'Need set \'outputs\' field.'): - l1.rpc.call('txprepare', {'destination': addr, 'feerate': 'slow'}) - - with pytest.raises(RpcError, match=r'Need set \'outputs\' field.'): - l1.rpc.call('txprepare', {'satoshi': Millisatoshi(amount * 100), 'feerate': '10perkw', 'minconf': 2}) - - l1.rpc.call('txprepare', {'destination': addr, 'satoshi': Millisatoshi(amount * 100), 'feerate': '2000perkw', 'minconf': 1}) - l1.rpc.call('txprepare', {'destination': addr, 'satoshi': Millisatoshi(amount * 100), 'feerate': '2000perkw'}) - l1.rpc.call('txprepare', {'destination': addr, 'satoshi': Millisatoshi(amount * 100)}) - - def test_txprepare_multi(node_factory, bitcoind): amount = 10000000 l1 = node_factory.get_node(random_hsm=True) @@ -880,60 +836,6 @@ def test_txsend(node_factory, bitcoind, chainparams): assert decode['vout'][changenum]['scriptPubKey']['addresses'][0] in [f['address'] for f in l1.rpc.listfunds()['outputs']] -def test_txprepare_restart(node_factory, bitcoind, chainparams): - amount = 1000000 - l1 = node_factory.get_node(may_fail=True) - addr = chainparams['example_addr'] - - # Add some funds to withdraw later: both bech32 and p2sh - for i in range(5): - bitcoind.rpc.sendtoaddress(l1.rpc.newaddr()['bech32'], - amount / 10**8) - bitcoind.rpc.sendtoaddress(l1.rpc.newaddr('p2sh-segwit')['p2sh-segwit'], - amount / 10**8) - bitcoind.generate_block(1) - wait_for(lambda: [o['status'] for o in l1.rpc.listfunds()['outputs']] == ['confirmed'] * 10) - - prep = l1.rpc.txprepare([{addr: 'all'}]) - decode = bitcoind.rpc.decoderawtransaction(prep['unsigned_tx']) - assert decode['txid'] == prep['txid'] - # All 10 inputs - assert len(decode['vin']) == 10 - - # L1 will forget all about it. - l1.restart() - - # It goes backwards in blockchain just in case there was a reorg. Wait. - wait_for(lambda: [o['status'] for o in l1.rpc.listfunds()['outputs']] == ['confirmed'] * 10) - - with pytest.raises(RpcError, match=r'not an unreleased txid'): - l1.rpc.txdiscard(prep['txid']) - - prep = l1.rpc.txprepare([{addr: 'all'}]) - - decode = bitcoind.rpc.decoderawtransaction(prep['unsigned_tx']) - assert decode['txid'] == prep['txid'] - # All 10 inputs - assert len(decode['vin']) == 10 - - # This will also work if we simply kill it. - l1.restart(clean=False) - - # It goes backwards in blockchain just in case there was a reorg. Wait. - wait_for(lambda: [o['status'] for o in l1.rpc.listfunds()['outputs']] == ['confirmed'] * 10) - - # It should have logged this for each output (any order) - template = r'wallet: reserved output {}/{} reset to available' - lines = [template.format(i['txid'], i['vout']) for i in decode['vin']] - l1.daemon.wait_for_logs(lines) - - prep = l1.rpc.txprepare([{addr: 'all'}]) - decode = bitcoind.rpc.decoderawtransaction(prep['unsigned_tx']) - assert decode['txid'] == prep['txid'] - # All 10 inputs - assert len(decode['vin']) == 10 - - @unittest.skipIf(TEST_NETWORK != 'regtest', "Fee outputs throw off our output matching logic") @unittest.skipIf(not EXPERIMENTAL_FEATURES, "Tests annotations which are compiled only with experimental features") def test_transaction_annotations(node_factory, bitcoind): From 6b2a3f4dfbb70b38fe4fd5bd318e481da72b9b46 Mon Sep 17 00:00:00 2001 From: Rusty Russell Date: Fri, 28 Aug 2020 12:07:57 +0930 Subject: [PATCH 12/30] txprepare: remove old code, switch to plugin. Some minor phrasing differences cause test changes. Signed-off-by: Rusty Russell Changelog-Changed: txprepare reservations stay across restarts: use fundpsbt/reservepsbt/unreservepsbt Changelog-Removed: txprepare `destination` `satoshi` argument form removed (deprecated v0.7.3) --- plugins/txprepare.c | 6 +- tests/test_connection.py | 6 +- tests/test_wallet.py | 24 +++++--- wallet/walletrpc.c | 120 --------------------------------------- 4 files changed, 22 insertions(+), 134 deletions(-) diff --git a/plugins/txprepare.c b/plugins/txprepare.c index 57e1d4964782..7aa322ba5050 100644 --- a/plugins/txprepare.c +++ b/plugins/txprepare.c @@ -434,21 +434,21 @@ static struct command_result *json_txsend(struct command *cmd, static const struct plugin_command commands[] = { { - "newtxprepare", + "txprepare", "bitcoin", "Create a transaction, with option to spend in future (either txsend and txdiscard)", "Create an unsigned transaction paying {outputs} with optional {feerate}, {minconf} and {utxos}", json_txprepare }, { - "newtxdiscard", + "txdiscard", "bitcoin", "Discard a transaction created by txprepare", "Discard a transcation by {txid}", json_txdiscard }, { - "newtxsend", + "txsend", "bitcoin", "Send a transaction created by txprepare", "Send a transacation by {txid}", diff --git a/tests/test_connection.py b/tests/test_connection.py index f7ab73ffba1d..1bdd72ac6866 100644 --- a/tests/test_connection.py +++ b/tests/test_connection.py @@ -771,7 +771,7 @@ def test_funding_fail(node_factory, bitcoind): l1.rpc.connect(l2.info['id'], 'localhost', l2.port) # We don't have enough left to cover fees if we try to spend it all. - with pytest.raises(RpcError, match=r'Cannot afford transaction'): + with pytest.raises(RpcError, match=r'Could not afford'): l1.rpc.fundchannel(l2.info['id'], funds) # Should still be connected. @@ -863,7 +863,7 @@ def test_funding_by_utxos(node_factory, bitcoind): utxos = [utxo["txid"] + ":" + str(utxo["output"]) for utxo in l1.rpc.listfunds()["outputs"]] # Fund with utxos we don't own - with pytest.raises(RpcError, match=r"No matching utxo was found from the wallet"): + with pytest.raises(RpcError, match=r"Unknown UTXO "): l3.rpc.fundchannel(l2.info["id"], int(0.01 * 10**8), utxos=utxos) # Fund with an empty array @@ -878,7 +878,7 @@ def test_funding_by_utxos(node_factory, bitcoind): l1.rpc.fundchannel(l3.info["id"], int(0.007 * 10**8), utxos=[utxos[2]]) # Fund another channel with already spent utxos - with pytest.raises(RpcError, match=r"No matching utxo was found from the wallet"): + with pytest.raises(RpcError, match=r"Already spent UTXO "): l1.rpc.fundchannel(l3.info["id"], int(0.01 * 10**8), utxos=utxos) diff --git a/tests/test_wallet.py b/tests/test_wallet.py index 3415f58322c8..53e94797071e 100644 --- a/tests/test_wallet.py +++ b/tests/test_wallet.py @@ -341,8 +341,7 @@ def test_txprepare(node_factory, bitcoind, chainparams): # Try passing unconfirmed utxos unconfirmed_utxo = l1.rpc.withdraw(l1.rpc.newaddr()["bech32"], 10**5) uutxos = [unconfirmed_utxo["txid"] + ":0"] - with pytest.raises(RpcError, match=r"Cannot afford transaction .* use " - "confirmed utxos."): + with pytest.raises(RpcError, match=r"Could not afford"): l1.rpc.txprepare([{addr: Millisatoshi(amount * 3.5 * 1000)}], utxos=uutxos) @@ -357,15 +356,24 @@ def test_txprepare(node_factory, bitcoind, chainparams): # We should have a change output, so this is exact assert len(decode['vout']) == 3 if chainparams['feeoutput'] else 2 - assert decode['vout'][1]['value'] == Decimal(amount * 3.5) / 10**8 - assert decode['vout'][1]['scriptPubKey']['type'] == 'witness_v0_keyhash' - assert decode['vout'][1]['scriptPubKey']['addresses'] == [addr] + # Change output pos is random. + for vout in decode['vout']: + if vout['scriptPubKey']['addresses'] == [addr]: + changeout = vout + + assert changeout['value'] == Decimal(amount * 3.5) / 10**8 + assert changeout['scriptPubKey']['type'] == 'witness_v0_keyhash' + assert changeout['scriptPubKey']['addresses'] == [addr] # Discard prep4 and get all funds again l1.rpc.txdiscard(prep5['txid']) - with pytest.raises(RpcError, match=r'this destination wants all satoshi. The count of outputs can\'t be more than 1'): - prep5 = l1.rpc.txprepare([{addr: Millisatoshi(amount * 3 * 1000)}, - {addr: 'all'}]) + # You can have one which is all, but not two. + prep5 = l1.rpc.txprepare([{addr: Millisatoshi(amount * 3 * 1000)}, + {addr: 'all'}]) + l1.rpc.txdiscard(prep5['txid']) + with pytest.raises(RpcError, match=r"'all'"): + prep5 = l1.rpc.txprepare([{addr: 'all'}, {addr: 'all'}]) + prep5 = l1.rpc.txprepare([{addr: Millisatoshi(amount * 3 * 500 + 100000)}, {addr: Millisatoshi(amount * 3 * 500 - 100000)}]) decode = bitcoind.rpc.decoderawtransaction(prep5['unsigned_tx']) diff --git a/wallet/walletrpc.c b/wallet/walletrpc.c index 453df3a10f40..278e30ea4d05 100644 --- a/wallet/walletrpc.c +++ b/wallet/walletrpc.c @@ -470,126 +470,6 @@ static struct command_result *json_prepare_tx(struct command *cmd, return NULL; } -static struct command_result *json_txprepare(struct command *cmd, - const char *buffer, - const jsmntok_t *obj UNNEEDED, - const jsmntok_t *params) -{ - struct unreleased_tx *utx; - struct command_result *res; - struct json_stream *response; - - res = json_prepare_tx(cmd, buffer, params, false, &utx, NULL); - if (res) - return res; - - /* utx will persist past this command. */ - tal_steal(cmd->ld->wallet, utx); - add_unreleased_tx(cmd->ld->wallet, utx); - - response = json_stream_success(cmd); - json_add_tx(response, "unsigned_tx", utx->tx); - json_add_txid(response, "txid", &utx->txid); - return command_success(cmd, response); -} -static const struct json_command txprepare_command = { - "txprepare", - "bitcoin", - json_txprepare, - "Create a transaction, with option to spend in future (either txsend and txdiscard)", - false -}; -AUTODATA(json_command, &txprepare_command); - -static struct command_result *param_unreleased_txid(struct command *cmd, - const char *name, - const char *buffer, - const jsmntok_t *tok, - struct unreleased_tx **utx) -{ - struct command_result *res; - struct bitcoin_txid *txid; - - res = param_txid(cmd, name, buffer, tok, &txid); - if (res) - return res; - - *utx = find_unreleased_tx(cmd->ld->wallet, txid); - if (!*utx) - return command_fail(cmd, LIGHTNINGD, - "%s not an unreleased txid", - type_to_string(cmd, struct bitcoin_txid, - txid)); - tal_free(txid); - return NULL; -} - -static struct command_result *json_txsend(struct command *cmd, - const char *buffer, - const jsmntok_t *obj UNNEEDED, - const jsmntok_t *params) -{ - struct unreleased_tx *utx; - - if (!param(cmd, buffer, params, - p_req("txid", param_unreleased_txid, &utx), - NULL)) - return command_param_failed(); - - /* We delete from list now, and this command owns it. */ - remove_unreleased_tx(utx); - tal_steal(cmd, utx); - - /* We're the owning cmd now. */ - utx->wtx->cmd = cmd; - - wallet_transaction_add(cmd->ld->wallet, utx->tx->wtx, 0, 0); - wallet_transaction_annotate(cmd->ld->wallet, &utx->txid, - TX_UNKNOWN, 0); - - return broadcast_and_wait(cmd, utx); -} - -static const struct json_command txsend_command = { - "txsend", - "bitcoin", - json_txsend, - "Sign and broadcast a transaction created by txprepare", - false -}; -AUTODATA(json_command, &txsend_command); - -static struct command_result *json_txdiscard(struct command *cmd, - const char *buffer, - const jsmntok_t *obj UNNEEDED, - const jsmntok_t *params) -{ - struct unreleased_tx *utx; - struct json_stream *response; - - if (!param(cmd, buffer, params, - p_req("txid", param_unreleased_txid, &utx), - NULL)) - return command_param_failed(); - - /* Free utx with this command */ - tal_steal(cmd, utx); - - response = json_stream_success(cmd); - json_add_tx(response, "unsigned_tx", utx->tx); - json_add_txid(response, "txid", &utx->txid); - return command_success(cmd, response); -} - -static const struct json_command txdiscard_command = { - "txdiscard", - "bitcoin", - json_txdiscard, - "Abandon a transaction created by txprepare", - false -}; -AUTODATA(json_command, &txdiscard_command); - /** * json_withdraw - Entrypoint for the withdrawal flow * From 182b761a467918a23319040a6d95a944e35b4897 Mon Sep 17 00:00:00 2001 From: Rusty Russell Date: Fri, 28 Aug 2020 12:08:57 +0930 Subject: [PATCH 13/30] walletrpc: remove now-unused txprepare parsing code. Signed-off-by: Rusty Russell --- wallet/walletrpc.c | 235 ++++----------------------------------------- 1 file changed, 18 insertions(+), 217 deletions(-) diff --git a/wallet/walletrpc.c b/wallet/walletrpc.c index 278e30ea4d05..c4c611252298 100644 --- a/wallet/walletrpc.c +++ b/wallet/walletrpc.c @@ -162,7 +162,7 @@ static struct command_result *broadcast_and_wait(struct command *cmd, return command_still_pending(cmd); } -/* Common code for withdraw and txprepare. +/* Parsing code for withdraw. * * Returns NULL on success, and fills in wtx, output and * maybe changekey (owned by cmd). Otherwise, cmd has failed, so don't @@ -170,18 +170,15 @@ static struct command_result *broadcast_and_wait(struct command *cmd, static struct command_result *json_prepare_tx(struct command *cmd, const char *buffer, const jsmntok_t *params, - bool for_withdraw, - struct unreleased_tx **utx, - u32 *feerate) + struct unreleased_tx **utx) { u32 *feerate_per_kw = NULL; struct command_result *result; u32 *minconf, maxheight; struct pubkey *changekey; struct bitcoin_tx_output **outputs; - const jsmntok_t *outputstok = NULL, *t; const u8 *destination = NULL; - size_t out_len, i; + size_t out_len; const struct utxo **chosen_utxos = NULL; u32 locktime; @@ -189,139 +186,16 @@ static struct command_result *json_prepare_tx(struct command *cmd, (*utx)->wtx = tal(*utx, struct wallet_tx); wtx_init(cmd, (*utx)->wtx, AMOUNT_SAT(-1ULL)); - if (!for_withdraw) { - /* From v0.7.3, the new style for *txprepare* use array of outputs - * to replace original 'destination' and 'satoshi' parameters.*/ - /* For generating help, give new-style. */ - if (!params || !deprecated_apis) { - if (!param(cmd, buffer, params, - p_req("outputs", param_array, &outputstok), - p_opt("feerate", param_feerate, &feerate_per_kw), - p_opt_def("minconf", param_number, &minconf, 1), - p_opt("utxos", param_utxos, &chosen_utxos), - NULL)) - return command_param_failed(); - } else if (params->type == JSMN_ARRAY) { - const jsmntok_t *firsttok, *secondtok, *thirdtok, *fourthtok; - - /* FIXME: This change completely destroyes the support for `check`. */ - if (!param(cmd, buffer, params, - p_req("outputs_or_destination", param_tok, &firsttok), - p_opt("feerate_or_sat", param_tok, &secondtok), - p_opt("minconf_or_feerate", param_tok, &thirdtok), - p_opt("utxos_or_minconf", param_tok, &fourthtok), - NULL)) - return command_param_failed(); - - if (firsttok->type == JSMN_ARRAY) { - /* New style: - * *txprepare* 'outputs' ['feerate'] ['minconf'] ['utxos'] */ - - /* outputs (required) */ - outputstok = firsttok; - - /* feerate (optional) */ - if (secondtok) { - result = param_feerate(cmd, "feerate", buffer, - secondtok, &feerate_per_kw); - if (result) - return result; - } - - /* minconf (optional) */ - if (thirdtok) { - result = param_number(cmd, "minconf", buffer, - thirdtok, &minconf); - if (result) - return result; - } else { - minconf = tal(cmd, u32); - *minconf = 1; - } - - /* utxos (optional) */ - if (fourthtok) { - result = param_utxos(cmd, "utxos", buffer, - fourthtok, &chosen_utxos); - if (result) - return result; - } - } else { - /* Old style: - * *txprepare* 'destination' 'satoshi' ['feerate'] ['minconf'] */ - - /* destination (required) */ - result = param_bitcoin_address(cmd, "destination", buffer, - firsttok, &destination); - if (result) - return result; - - /* satoshi (required) */ - if (!secondtok) - return command_fail(cmd, JSONRPC2_INVALID_PARAMS, - "Need set 'satoshi' field."); - result = param_wtx(cmd, "satoshi", buffer, - secondtok, (*utx)->wtx); - if (result) - return result; - - /* feerate (optional) */ - if (thirdtok) { - result = param_feerate(cmd, "feerate", buffer, - thirdtok, &feerate_per_kw); - if (result) - return result; - } - - /* minconf (optional) */ - if (fourthtok) { - result = param_number(cmd, "minconf", buffer, - fourthtok, &minconf); - if (result) - return result; - } else { - minconf = tal(cmd, u32); - *minconf = 1; - } - } - } else { - const jsmntok_t *satoshitok = NULL; - if (!param(cmd, buffer, params, - p_opt("outputs", param_array, &outputstok), - p_opt("destination", param_bitcoin_address, - &destination), - p_opt("satoshi", param_tok, &satoshitok), - p_opt("feerate", param_feerate, &feerate_per_kw), - p_opt_def("minconf", param_number, &minconf, 1), - p_opt("utxos", param_utxos, &chosen_utxos), - NULL)) - /* If the parameters mixed the new style and the old style, - * fail it. */ - return command_param_failed(); - - if (!outputstok) { - if (!destination || !satoshitok) - return command_fail(cmd, JSONRPC2_INVALID_PARAMS, - "Need set 'outputs' field."); - - result = param_wtx(cmd, "satoshi", buffer, - satoshitok, (*utx)->wtx); - if (result) - return result; - } - } - } else { - /* *withdraw* command still use 'destination' and 'satoshi' as parameters. */ - if (!param(cmd, buffer, params, - p_req("destination", param_bitcoin_address, - &destination), - p_req("satoshi", param_wtx, (*utx)->wtx), - p_opt("feerate", param_feerate, &feerate_per_kw), - p_opt_def("minconf", param_number, &minconf, 1), - p_opt("utxos", param_utxos, &chosen_utxos), - NULL)) - return command_param_failed(); - } + /* *withdraw* command still use 'destination' and 'satoshi' as parameters. */ + if (!param(cmd, buffer, params, + p_req("destination", param_bitcoin_address, + &destination), + p_req("satoshi", param_wtx, (*utx)->wtx), + p_opt("feerate", param_feerate, &feerate_per_kw), + p_opt_def("minconf", param_number, &minconf, 1), + p_opt("utxos", param_utxos, &chosen_utxos), + NULL)) + return command_param_failed(); /* Setting the locktime to the next block to be mined has multiple * benefits: @@ -348,81 +222,11 @@ static struct command_result *json_prepare_tx(struct command *cmd, maxheight = minconf_to_maxheight(*minconf, cmd->ld); - /* *withdraw* command or old *txprepare* command. - * Support only one output. */ - if (destination) { - outputs = tal_arr(tmpctx, struct bitcoin_tx_output *, 1); - outputs[0] = new_tx_output(outputs, (*utx)->wtx->amount, - destination); - out_len = tal_count(outputs[0]->script); - - goto create_tx; - } - - if (outputstok->size == 0) - return command_fail(cmd, JSONRPC2_INVALID_PARAMS, "Empty outputs"); - - outputs = tal_arr(tmpctx, struct bitcoin_tx_output *, outputstok->size); - out_len = 0; - (*utx)->wtx->all_funds = false; - (*utx)->wtx->amount = AMOUNT_SAT(0); - json_for_each_arr(i, t, outputstok) { - struct amount_sat *amount; - const u8 *destination; - enum address_parse_result res; - - /* output format: {destination: amount} */ - if (t->type != JSMN_OBJECT) - return command_fail(cmd, JSONRPC2_INVALID_PARAMS, - "The output format must be " - "{destination: amount}"); - - res = json_to_address_scriptpubkey(cmd, - chainparams, - buffer, &t[1], - &destination); - if (res == ADDRESS_PARSE_UNRECOGNIZED) - return command_fail(cmd, JSONRPC2_INVALID_PARAMS, - "Could not parse destination address"); - else if (res == ADDRESS_PARSE_WRONG_NETWORK) - return command_fail(cmd, JSONRPC2_INVALID_PARAMS, - "Destination address is not on network %s", - chainparams->network_name); + outputs = tal_arr(tmpctx, struct bitcoin_tx_output *, 1); + outputs[0] = new_tx_output(outputs, (*utx)->wtx->amount, + destination); + out_len = tal_count(outputs[0]->script); - amount = tal(tmpctx, struct amount_sat); - if (!json_to_sat_or_all(buffer, &t[2], amount)) - return command_fail(cmd, JSONRPC2_INVALID_PARAMS, - "'%.*s' is a invalid satoshi amount", - t[2].end - t[2].start, buffer + t[2].start); - - outputs[i] = new_tx_output(outputs, *amount, - cast_const(u8 *, destination)); - out_len += tal_count(destination); - - /* In fact, the maximum amount of bitcoin satoshi is 2.1e15. - * It can't be equal to/bigger than 2^64. - * On the hand, the maximum amount of litoshi is 8.4e15, - * which also can't overflow. */ - /* This means this destination need "all" satoshi we have. */ - if (amount_sat_eq(*amount, AMOUNT_SAT(-1ULL))) { - if (outputstok->size > 1) - return command_fail(cmd, JSONRPC2_INVALID_PARAMS, - "outputs[%zi]: this destination wants" - " all satoshi. The count of outputs" - " can't be more than 1. ", i); - (*utx)->wtx->all_funds = true; - /* `AMOUNT_SAT(-1ULL)` is the max permissible for `wallet_select_all`. */ - (*utx)->wtx->amount = *amount; - break; - } - - if (!amount_sat_add(&(*utx)->wtx->amount, (*utx)->wtx->amount, *amount)) - return command_fail(cmd, JSONRPC2_INVALID_PARAMS, - "outputs: The sum of first %zi outputs" - " overflow. ", i); - } - -create_tx: if (chosen_utxos) result = wtx_from_utxos((*utx)->wtx, *feerate_per_kw, out_len, maxheight, @@ -464,9 +268,6 @@ static struct command_result *json_prepare_tx(struct command *cmd, locktime); bitcoin_txid((*utx)->tx, &(*utx)->txid); - - if (feerate) - *feerate = *feerate_per_kw; return NULL; } @@ -485,7 +286,7 @@ static struct command_result *json_withdraw(struct command *cmd, struct unreleased_tx *utx; struct command_result *res; - res = json_prepare_tx(cmd, buffer, params, true, &utx, NULL); + res = json_prepare_tx(cmd, buffer, params, &utx); if (res) return res; From c319f7697ee7de5442e897779e19603c05b5da7a Mon Sep 17 00:00:00 2001 From: Rusty Russell Date: Fri, 28 Aug 2020 12:09:57 +0930 Subject: [PATCH 14/30] wallet: wean fundpsbt off tx_spending_utxos. We only need a subset of its functionality, so reproduce that here. Signed-off-by: Rusty Russell --- wallet/reservation.c | 75 ++++++++++++++++++++++++++++++++++++++------ 1 file changed, 66 insertions(+), 9 deletions(-) diff --git a/wallet/reservation.c b/wallet/reservation.c index 7680614f03be..4d2474434b0f 100644 --- a/wallet/reservation.c +++ b/wallet/reservation.c @@ -1,7 +1,11 @@ /* Dealing with reserving UTXOs */ +#include +#include +#include #include #include #include +#include #include #include #include @@ -208,6 +212,64 @@ static bool inputs_sufficient(struct amount_sat input, return false; } +static struct wally_psbt *psbt_using_utxos(const tal_t *ctx, + struct utxo **utxos, + const struct ext_key *bip32_base, + u32 nlocktime, + u32 nsequence) +{ + struct pubkey key; + u8 *scriptSig, *scriptPubkey, *redeemscript; + struct bitcoin_tx *tx; + + /* FIXME: Currently the easiest way to get a PSBT is via a tx */ + tx = bitcoin_tx(ctx, chainparams, tal_count(utxos), 0, nlocktime); + + for (size_t i = 0; i < tal_count(utxos); i++) { + u32 this_nsequence; + + if (utxos[i]->is_p2sh) { + bip32_pubkey(bip32_base, &key, utxos[i]->keyindex); + scriptSig = bitcoin_scriptsig_p2sh_p2wpkh(tmpctx, &key); + redeemscript = bitcoin_redeem_p2sh_p2wpkh(tmpctx, &key); + scriptPubkey = scriptpubkey_p2sh(tmpctx, redeemscript); + + /* Make sure we've got the right info! */ + if (utxos[i]->scriptPubkey) + assert(memeq(utxos[i]->scriptPubkey, + tal_bytelen(utxos[i]->scriptPubkey), + scriptPubkey, tal_bytelen(scriptPubkey))); + } else { + scriptSig = NULL; + redeemscript = NULL; + scriptPubkey = utxos[i]->scriptPubkey; + } + + /* BOLT-a12da24dd0102c170365124782b46d9710950ac1 #3: + * #### `to_remote` Output + * ... + * The output is spent by a transaction with `nSequence` field + * set to `1` and witness: + */ + if (utxos[i]->close_info + && utxos[i]->close_info->option_anchor_outputs) + this_nsequence = 1; + else + this_nsequence = nsequence; + + bitcoin_tx_add_input(tx, &utxos[i]->txid, utxos[i]->outnum, + this_nsequence, scriptSig, utxos[i]->amount, + scriptPubkey, NULL); + + /* Add redeemscript to the PSBT input */ + if (redeemscript) + psbt_input_set_redeemscript(tx->psbt, i, redeemscript); + + } + + return tx->psbt; +} + static struct command_result *finish_psbt(struct command *cmd, struct utxo **utxos, u32 feerate_per_kw, @@ -217,7 +279,7 @@ static struct command_result *finish_psbt(struct command *cmd, u32 *locktime) { struct json_stream *response; - struct bitcoin_tx *tx; + struct wally_psbt *psbt; u32 current_height = get_block_height(cmd->ld->topology); /* Setting the locktime to the next block to be mined has multiple @@ -238,16 +300,11 @@ static struct command_result *finish_psbt(struct command *cmd, *locktime -= pseudorand(100); } - /* FIXME: tx_spending_utxos does more than we need, but there - * are other users right now. */ - tx = tx_spending_utxos(cmd, chainparams, - cast_const2(const struct utxo **, utxos), - cmd->ld->wallet->bip32_base, - false, 0, *locktime, - BITCOIN_TX_RBF_SEQUENCE); + psbt = psbt_using_utxos(cmd, utxos, cmd->ld->wallet->bip32_base, + *locktime, BITCOIN_TX_RBF_SEQUENCE); response = json_stream_success(cmd); - json_add_psbt(response, "psbt", tx->psbt); + json_add_psbt(response, "psbt", psbt); json_add_num(response, "feerate_per_kw", feerate_per_kw); json_add_num(response, "estimated_final_weight", weight); json_add_amount_sat_only(response, "excess_msat", excess); From 3b8c0a7397cc0694117dfe63ecb2bff0b6957a4b Mon Sep 17 00:00:00 2001 From: Rusty Russell Date: Fri, 28 Aug 2020 12:10:57 +0930 Subject: [PATCH 15/30] sendpsbt: just reserve (maybe bump) inputs on send, don't mark spent. Marking spent means if the transaction doesn't confirm for some reason, the user will need to force a rescan to find the funds. Now we have timed reservations, reserving for (an additional) 12 hours should be sufficient. We also take this opportunity (now we have our own callback path) to record the tx in the wallet only on success. Signed-off-by: Rusty Russell --- tests/test_closing.py | 2 +- tests/test_connection.py | 24 +++++++++++- wallet/wallet.c | 8 ++-- wallet/walletrpc.c | 80 +++++++++++++++++++++++++++++----------- 4 files changed, 87 insertions(+), 27 deletions(-) diff --git a/tests/test_closing.py b/tests/test_closing.py index 96d0f4075616..3ae387b36016 100644 --- a/tests/test_closing.py +++ b/tests/test_closing.py @@ -1543,7 +1543,7 @@ def test_listfunds_after_their_unilateral(node_factory, bitcoind): channel_id = first_channel_id(l1, l2) # listfunds will show 1 output change, and channels. - assert len(l1.rpc.listfunds()['outputs']) == 1 + assert len([o for o in l1.rpc.listfunds()['outputs'] if not o['reserved']]) == 1 l1.stop() l2.rpc.close(l1.info['id'], unilateraltimeout=1) diff --git a/tests/test_connection.py b/tests/test_connection.py index 1bdd72ac6866..a0551d2c4b73 100644 --- a/tests/test_connection.py +++ b/tests/test_connection.py @@ -703,6 +703,8 @@ def test_funding_change(node_factory, bitcoind): assert only_one(outputs)['value'] == 10000000 l1.rpc.fundchannel(l2.info['id'], 1000000) + bitcoind.generate_block(1, wait_for_mempool=1) + sync_blockheight(bitcoind, [l1]) outputs = {r['status']: r['value'] for r in l1.db_query( 'SELECT status, SUM(value) AS value FROM outputs GROUP BY status;')} @@ -734,10 +736,21 @@ def test_funding_all_too_much(node_factory): """ l1, l2 = node_factory.line_graph(2, fundchannel=False) - l1.fundwallet(2**24 + 10000) + addr, txid = l1.fundwallet(2**24 + 10000) l1.rpc.fundchannel(l2.info['id'], "all") - assert only_one(l1.rpc.listfunds()['outputs'])['status'] == 'unconfirmed' + # One reserved, confirmed output spent above, and one change. + outputs = l1.rpc.listfunds()['outputs'] + + spent = only_one([o for o in outputs if o['status'] == 'confirmed']) + + assert spent['txid'] == txid + assert spent['address'] == addr + assert spent['reserved'] is True + + pending = only_one([o for o in outputs if o['status'] != 'confirmed']) + assert pending['status'] == 'unconfirmed' + assert pending['reserved'] is False assert only_one(l1.rpc.listfunds()['channels'])['channel_total_sat'] == 2**24 - 1 @@ -877,6 +890,13 @@ def test_funding_by_utxos(node_factory, bitcoind): l1.rpc.connect(l3.info["id"], "localhost", l3.port) l1.rpc.fundchannel(l3.info["id"], int(0.007 * 10**8), utxos=[utxos[2]]) + # Fund another channel with already reserved utxos + with pytest.raises(RpcError, match=r"UTXO.*already reserved"): + l1.rpc.fundchannel(l3.info["id"], int(0.01 * 10**8), utxos=utxos) + + bitcoind.generate_block(1, wait_for_mempool=1) + sync_blockheight(bitcoind, [l1]) + # Fund another channel with already spent utxos with pytest.raises(RpcError, match=r"Already spent UTXO "): l1.rpc.fundchannel(l3.info["id"], int(0.01 * 10**8), utxos=utxos) diff --git a/wallet/wallet.c b/wallet/wallet.c index 15594ad82e5f..dec065ff35cc 100644 --- a/wallet/wallet.c +++ b/wallet/wallet.c @@ -3208,13 +3208,15 @@ wallet_outpoint_spend(struct wallet *w, const tal_t *ctx, const u32 blockheight, int changes; if (outpointfilter_matches(w->owned_outpoints, txid, outnum)) { stmt = db_prepare_v2(w->db, SQL("UPDATE outputs " - "SET spend_height = ? " + "SET spend_height = ?, " + " status = ? " "WHERE prev_out_tx = ?" " AND prev_out_index = ?")); db_bind_int(stmt, 0, blockheight); - db_bind_sha256d(stmt, 1, &txid->shad); - db_bind_int(stmt, 2, outnum); + db_bind_int(stmt, 1, output_status_in_db(output_state_spent)); + db_bind_sha256d(stmt, 2, &txid->shad); + db_bind_int(stmt, 3, outnum); db_exec_prepared_v2(take(stmt)); diff --git a/wallet/walletrpc.c b/wallet/walletrpc.c index c4c611252298..2b2db663c3d1 100644 --- a/wallet/walletrpc.c +++ b/wallet/walletrpc.c @@ -1043,25 +1043,69 @@ static const struct json_command signpsbt_command = { AUTODATA(json_command, &signpsbt_command); +struct sending_psbt { + struct command *cmd; + struct utxo **utxos; + struct wally_tx *wtx; +}; + +static void sendpsbt_done(struct bitcoind *bitcoind UNUSED, + bool success, const char *msg, + struct sending_psbt *sending) +{ + struct lightningd *ld = sending->cmd->ld; + struct json_stream *response; + struct bitcoin_txid txid; + struct amount_sat change; + + if (!success) { + /* Unreserve the inputs again. */ + for (size_t i = 0; i < tal_count(sending->utxos); i++) { + wallet_unreserve_utxo(ld->wallet, + sending->utxos[i], + get_block_height(ld->topology)); + } + + was_pending(command_fail(sending->cmd, LIGHTNINGD, + "Error broadcasting transaction: %s." + " Unsent tx discarded %s", + msg, + type_to_string(tmpctx, struct wally_tx, + sending->wtx))); + return; + } + + wallet_transaction_add(ld->wallet, sending->wtx, 0, 0); + + /* Extract the change output and add it to the DB */ + wallet_extract_owned_outputs(ld->wallet, sending->wtx, NULL, &change); + + response = json_stream_success(sending->cmd); + wally_txid(sending->wtx, &txid); + json_add_hex_talarr(response, "tx", linearize_wtx(tmpctx, sending->wtx)); + json_add_txid(response, "txid", &txid); + was_pending(command_success(sending->cmd, response)); +} + static struct command_result *json_sendpsbt(struct command *cmd, const char *buffer, const jsmntok_t *obj UNNEEDED, const jsmntok_t *params) { struct command_result *res; + struct sending_psbt *sending; struct wally_psbt *psbt; - struct wally_tx *w_tx; - struct tx_broadcast *txb; - struct utxo **utxos; - struct bitcoin_txid txid; + struct lightningd *ld = cmd->ld; if (!param(cmd, buffer, params, p_req("psbt", param_psbt, &psbt), NULL)) return command_param_failed(); - w_tx = psbt_finalize(psbt, true); - if (!w_tx) + sending = tal(cmd, struct sending_psbt); + sending->cmd = cmd; + sending->wtx = tal_steal(sending, psbt_finalize(psbt, true)); + if (!sending->wtx) return command_fail(cmd, LIGHTNINGD, "PSBT not finalizeable %s", type_to_string(tmpctx, struct wally_psbt, @@ -1070,27 +1114,21 @@ static struct command_result *json_sendpsbt(struct command *cmd, /* We have to find/locate the utxos that are ours on this PSBT, * so that we know who to mark as used. */ - res = match_psbt_inputs_to_utxos(cmd, psbt, NULL, &utxos); + res = match_psbt_inputs_to_utxos(cmd, psbt, NULL, &sending->utxos); if (res) return res; - txb = tal(cmd, struct tx_broadcast); - txb->utxos = cast_const2(const struct utxo **, - tal_steal(txb, utxos)); - txb->wtx = tal_steal(txb, w_tx); - txb->cmd = cmd; - txb->expected_change = NULL; - - /* FIXME: Do this *after* successful broadcast! */ - wallet_transaction_add(cmd->ld->wallet, txb->wtx, 0, 0); - wally_txid(txb->wtx, &txid); - wallet_transaction_annotate(cmd->ld->wallet, &txid, - TX_UNKNOWN, 0); + for (size_t i = 0; i < tal_count(sending->utxos); i++) { + if (!wallet_reserve_utxo(ld->wallet, sending->utxos[i], + get_block_height(ld->topology))) + fatal("UTXO not reservable?"); + } /* Now broadcast the transaction */ bitcoind_sendrawtx(cmd->ld->topology->bitcoind, - tal_hex(tmpctx, linearize_wtx(tmpctx, w_tx)), - wallet_withdrawal_broadcast, txb); + tal_hex(tmpctx, + linearize_wtx(tmpctx, sending->wtx)), + sendpsbt_done, sending); return command_still_pending(cmd); } From 8d4557938bff5ceb0449f4dd522509191d2cbe6c Mon Sep 17 00:00:00 2001 From: Rusty Russell Date: Fri, 28 Aug 2020 12:11:57 +0930 Subject: [PATCH 16/30] plugins/txprepare: move functions higher. Simple moveonly change for next patch. Signed-off-by: Rusty Russell --- plugins/txprepare.c | 96 ++++++++++++++++++++++----------------------- 1 file changed, 48 insertions(+), 48 deletions(-) diff --git a/plugins/txprepare.c b/plugins/txprepare.c index 7aa322ba5050..62fc5105e841 100644 --- a/plugins/txprepare.c +++ b/plugins/txprepare.c @@ -117,6 +117,54 @@ static struct command_result *param_outputs(struct command *cmd, return NULL; } +/* Called after lightningd has broadcast the transaction. */ +static struct command_result *sendpsbt_done(struct command *cmd, + const char *buf, + const jsmntok_t *result, + struct unreleased_tx *utx) +{ + struct json_stream *out; + + out = jsonrpc_stream_success(cmd); + json_add_hex_talarr(out, "tx", linearize_wtx(tmpctx, utx->tx)); + json_add_txid(out, "txid", &utx->txid); + return command_finished(cmd, out); +} + +/* Called after lightningd has signed the inputs. */ +static struct command_result *signpsbt_done(struct command *cmd, + const char *buf, + const jsmntok_t *result, + struct unreleased_tx *utx) +{ + struct out_req *req; + const jsmntok_t *psbttok = json_get_member(buf, result, "signed_psbt"); + struct bitcoin_txid txid; + + tal_free(utx->psbt); + utx->psbt = json_tok_psbt(utx, buf, psbttok); + /* Replace with signed tx. */ + tal_free(utx->tx); + + /* The txid from the final should match our expectation. */ + psbt_txid(utx->psbt, &txid, &utx->tx); + if (!bitcoin_txid_eq(&txid, &utx->txid)) { + return command_fail(cmd, LIGHTNINGD, + "Signed tx changed txid? Had '%s' now '%s'", + tal_hex(tmpctx, + linearize_wtx(tmpctx, utx->tx)), + tal_hex(tmpctx, + linearize_wtx(tmpctx, + utx->psbt->tx))); + } + + req = jsonrpc_request_start(cmd->plugin, cmd, "sendpsbt", + sendpsbt_done, forward_error, + utx); + json_add_psbt(req->js, "psbt", utx->psbt); + return send_outreq(cmd->plugin, req); +} + static struct command_result *finish_txprepare(struct command *cmd, struct txprepare *txp) { @@ -360,54 +408,6 @@ static struct command_result *json_txdiscard(struct command *cmd, return send_outreq(cmd->plugin, req); } -/* Called after lightningd has broadcast the transaction. */ -static struct command_result *sendpsbt_done(struct command *cmd, - const char *buf, - const jsmntok_t *result, - struct unreleased_tx *utx) -{ - struct json_stream *out; - - out = jsonrpc_stream_success(cmd); - json_add_hex_talarr(out, "tx", linearize_wtx(tmpctx, utx->tx)); - json_add_txid(out, "txid", &utx->txid); - return command_finished(cmd, out); -} - -/* Called after lightningd has signed the inputs. */ -static struct command_result *signpsbt_done(struct command *cmd, - const char *buf, - const jsmntok_t *result, - struct unreleased_tx *utx) -{ - struct out_req *req; - const jsmntok_t *psbttok = json_get_member(buf, result, "signed_psbt"); - struct bitcoin_txid txid; - - tal_free(utx->psbt); - utx->psbt = json_tok_psbt(utx, buf, psbttok); - /* Replace with signed tx. */ - tal_free(utx->tx); - - /* The txid from the final should match our expectation. */ - psbt_txid(utx->psbt, &txid, &utx->tx); - if (!bitcoin_txid_eq(&txid, &utx->txid)) { - return command_fail(cmd, LIGHTNINGD, - "Signed tx changed txid? Had '%s' now '%s'", - tal_hex(tmpctx, - linearize_wtx(tmpctx, utx->tx)), - tal_hex(tmpctx, - linearize_wtx(tmpctx, - utx->psbt->tx))); - } - - req = jsonrpc_request_start(cmd->plugin, cmd, "sendpsbt", - sendpsbt_done, forward_error, - utx); - json_add_psbt(req->js, "psbt", utx->psbt); - return send_outreq(cmd->plugin, req); -} - static struct command_result *json_txsend(struct command *cmd, const char *buffer, const jsmntok_t *params) From f34d0b1cf5c0fc38c3775dde8556fb832698bd81 Mon Sep 17 00:00:00 2001 From: Rusty Russell Date: Fri, 28 Aug 2020 12:12:57 +0930 Subject: [PATCH 17/30] plugins/txprepare: create simple variant for "withdraw". This is a little lazy, but simpler than extracting the common parts or making withdraw a plugin which calls txprepare (which should be deprecated soon in favor of fundpsbt etc). Signed-off-by: Rusty Russell --- plugins/txprepare.c | 101 ++++++++++++++++++++++++++++++++++++++------ 1 file changed, 87 insertions(+), 14 deletions(-) diff --git a/plugins/txprepare.c b/plugins/txprepare.c index 62fc5105e841..bf587e113465 100644 --- a/plugins/txprepare.c +++ b/plugins/txprepare.c @@ -37,6 +37,9 @@ struct txprepare { /* Once we have reserved all the inputs, this is set. */ struct amount_sat change_amount; + + /* For withdraw, we actually send immediately. */ + bool is_withdraw; }; struct unreleased_tx { @@ -191,8 +194,19 @@ static struct command_result *finish_txprepare(struct command *cmd, utx = tal(NULL, struct unreleased_tx); utx->psbt = tal_steal(utx, txp->psbt); psbt_txid(txp->psbt, &utx->txid, &utx->tx); - list_add(&unreleased_txs, &utx->list); + /* If this is a withdraw, we sign and send immediately. */ + if (txp->is_withdraw) { + struct out_req *req; + + req = jsonrpc_request_start(cmd->plugin, cmd, "signpsbt", + signpsbt_done, forward_error, + utx); + json_add_psbt(req->js, "psbt", utx->psbt); + return send_outreq(cmd->plugin, req); + } + + list_add(&unreleased_txs, &utx->list); out = jsonrpc_stream_success(cmd); json_add_hex_talarr(out, "unsigned_tx", linearize_wtx(tmpctx, utx->tx)); json_add_txid(out, "txid", &utx->txid); @@ -300,22 +314,17 @@ static struct command_result *psbt_created(struct command *cmd, return send_outreq(cmd->plugin, req); } -static struct command_result *json_txprepare(struct command *cmd, - const char *buffer, - const jsmntok_t *params) +/* Common point for txprepare and withdraw */ +static struct command_result *txprepare_continue(struct command *cmd, + struct txprepare *txp, + const char *feerate, + unsigned int *minconf, + const char *utxos, + bool is_withdraw) { - struct txprepare *txp = tal(cmd, struct txprepare); struct out_req *req; - const char *feerate, *utxos; - unsigned int *minconf; - if (!param(cmd, buffer, params, - p_req("outputs", param_outputs, txp), - p_opt("feerate", param_string, &feerate), - p_opt_def("minconf", param_number, &minconf, 1), - p_opt("utxos", param_string, &utxos), - NULL)) - return command_param_failed(); + txp->is_withdraw = is_withdraw; /* p_opt_def doesn't compile with strings... */ if (!feerate) @@ -346,6 +355,25 @@ static struct command_result *json_txprepare(struct command *cmd, return send_outreq(cmd->plugin, req); } +static struct command_result *json_txprepare(struct command *cmd, + const char *buffer, + const jsmntok_t *params) +{ + struct txprepare *txp = tal(cmd, struct txprepare); + const char *feerate, *utxos; + unsigned int *minconf; + + if (!param(cmd, buffer, params, + p_req("outputs", param_outputs, txp), + p_opt("feerate", param_string, &feerate), + p_opt_def("minconf", param_number, &minconf, 1), + p_opt("utxos", param_string, &utxos), + NULL)) + return command_param_failed(); + + return txprepare_continue(cmd, txp, feerate, minconf, utxos, false); +} + /* Called after we've unreserved the inputs. */ static struct command_result *unreserve_done(struct command *cmd, const char *buf, @@ -432,6 +460,44 @@ static struct command_result *json_txsend(struct command *cmd, return send_outreq(cmd->plugin, req); } +static struct command_result *json_withdraw(struct command *cmd, + const char *buffer, + const jsmntok_t *params) +{ + struct txprepare *txp = tal(cmd, struct txprepare); + struct amount_sat *amount; + const u8 *scriptpubkey; + const char *feerate, *utxos; + unsigned int *minconf; + + if (!param(cmd, buffer, params, + p_req("destination", param_bitcoin_address, + &scriptpubkey), + p_req("satoshi", param_sat_or_all, &amount), + p_opt("feerate", param_string, &feerate), + p_opt_def("minconf", param_number, &minconf, 1), + p_opt("utxos", param_string, &utxos), + NULL)) + return command_param_failed(); + + /* Convert destination/satoshi into array as txprepare expects */ + txp->outputs = tal_arr(txp, struct tx_output, 1); + + if (amount_sat_eq(*amount, AMOUNT_SAT(-1ULL))) { + txp->all_output_idx = 0; + txp->output_total = AMOUNT_SAT(0); + } else { + txp->all_output_idx = -1; + txp->output_total = *amount; + } + txp->outputs[0].amount = *amount; + txp->outputs[0].script = scriptpubkey; + txp->weight = bitcoin_tx_core_weight(1, tal_count(txp->outputs)) + + bitcoin_tx_output_weight(tal_bytelen(scriptpubkey)); + + return txprepare_continue(cmd, txp, feerate, minconf, utxos, true); +} + static const struct plugin_command commands[] = { { "txprepare", @@ -454,6 +520,13 @@ static const struct plugin_command commands[] = { "Send a transacation by {txid}", json_txsend }, + { + "newwithdraw", + "bitcoin", + "Send funds to {destination} address", + "Send to {destination} {satoshi} (or 'all') at optional {feerate} using utxos from {minconf} or {utxos}.", + json_withdraw + }, }; int main(int argc, char *argv[]) From e5f7e5b5771b94040e563816f4153fc1327062c1 Mon Sep 17 00:00:00 2001 From: Rusty Russell Date: Fri, 28 Aug 2020 12:13:57 +0930 Subject: [PATCH 18/30] pytest: allow variable-order coin_moves, and give more information on failure. Signed-off-by: Rusty Russell --- tests/utils.py | 40 +++++++++++++++++++++++++++++++++------- 1 file changed, 33 insertions(+), 7 deletions(-) diff --git a/tests/utils.py b/tests/utils.py index 565cdd8bc3f6..ea6bb6a72724 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -54,6 +54,18 @@ def expected_channel_features(wumbo_channels=False, extra=[]): return hex_bits(features + extra) +def move_matches(exp, mv): + if mv['type'] != exp['type']: + return False + if mv['credit'] != "{}msat".format(exp['credit']): + return False + if mv['debit'] != "{}msat".format(exp['debit']): + return False + if mv['tag'] != exp['tag']: + return False + return True + + def check_coin_moves(n, account_id, expected_moves, chainparams): moves = n.rpc.call('listcoinmoves_plugin')['coin_moves'] node_id = n.info['id'] @@ -64,21 +76,35 @@ def check_coin_moves(n, account_id, expected_moves, chainparams): Millisatoshi(mv['credit']).millisatoshis, Millisatoshi(mv['debit']).millisatoshis, mv['tag'])) - - assert len(acct_moves) == len(expected_moves) - for mv, exp in list(zip(acct_moves, expected_moves)): assert mv['version'] == 1 assert mv['node_id'] == node_id - assert mv['type'] == exp['type'] - assert mv['credit'] == "{}msat".format(exp['credit']) - assert mv['debit'] == "{}msat".format(exp['debit']) - assert mv['tag'] == exp['tag'] assert mv['timestamp'] > 0 assert mv['coin_type'] == chainparams['bip173_prefix'] # chain moves should have blockheights if mv['type'] == 'chain_mvt': assert mv['blockheight'] is not None + for num, m in enumerate(expected_moves): + # They can group things which are in any order. + if isinstance(m, list): + number_moves = len(m) + for acct_move in acct_moves[:number_moves]: + found = None + for i in range(len(m)): + if move_matches(m[i], acct_move): + found = i + break + if found is None: + raise ValueError("Unexpected move {} amongst {}".format(acct_move, m)) + del m[i] + acct_moves = acct_moves[number_moves:] + else: + if not move_matches(m, acct_moves[0]): + raise ValueError("Unexpected move {}: {} != {}".format(num, acct_moves[0], m)) + acct_moves = acct_moves[1:] + + assert acct_moves == [] + def check_coin_moves_idx(n): """ Just check that the counter increments smoothly""" From 2696ec6ccb15b2a0c280ad3f6ea29e122f48b4d4 Mon Sep 17 00:00:00 2001 From: Rusty Russell Date: Fri, 28 Aug 2020 12:14:57 +0930 Subject: [PATCH 19/30] pytest: fix assumptions in test_withdraw_misc First, simplify: amount is set to 1000000, but then we deposit 1000000 + 0.01btc (i.e. 2000000), and we always use 2 * amount. Just use a single constant to make it clear. Secondly, we assume that the wallet considers outputs spent as soon as we created the tx: this will not be true once withdraw uses sendpsbt. So, we generate blocks, but now sometimes withdraw will pick up change txs, so we need to reserve them to avoid that messing our coinmovements. Finally, we assumed the withdrawl order was BIP69, which becomes variable. Signed-off-by: Rusty Russell --- tests/test_misc.py | 102 ++++++++++++++++++++++++++++--------------- tests/test_plugin.py | 14 ++++-- 2 files changed, 78 insertions(+), 38 deletions(-) diff --git a/tests/test_misc.py b/tests/test_misc.py index 4a70e2f271db..8caaa5c326b6 100644 --- a/tests/test_misc.py +++ b/tests/test_misc.py @@ -486,10 +486,17 @@ def is_p2wpkh(output): def test_withdraw_misc(node_factory, bitcoind, chainparams): + def dont_spend_outputs(n, txid): + """Reserve both outputs (we assume there are two!) in case any our ours, so we don't spend change: wrecks accounting checks""" + n.rpc.reserveinputs(bitcoind.rpc.createpsbt([{'txid': txid, + 'vout': 0}, + {'txid': txid, + 'vout': 1}], [])) + # We track channel balances, to verify that accounting is ok. coin_mvt_plugin = os.path.join(os.getcwd(), 'tests/plugins/coin_movements.py') - amount = 1000000 + amount = 2000000 # Don't get any funds from previous runs. l1 = node_factory.get_node(random_hsm=True, options={'plugin': coin_mvt_plugin}, @@ -499,7 +506,7 @@ def test_withdraw_misc(node_factory, bitcoind, chainparams): # Add some funds to withdraw later for i in range(10): - l1.bitcoin.rpc.sendtoaddress(addr, amount / 10**8 + 0.01) + l1.bitcoin.rpc.sendtoaddress(addr, amount / 10**8) bitcoind.generate_block(1) wait_for(lambda: len(l1.rpc.listfunds()['outputs']) == 10) @@ -518,7 +525,7 @@ def test_withdraw_misc(node_factory, bitcoind, chainparams): with pytest.raises(RpcError, match=r'Cannot afford transaction'): l1.rpc.withdraw(waddr, amount * 100) - out = l1.rpc.withdraw(waddr, 2 * amount) + out = l1.rpc.withdraw(waddr, amount) # Make sure bitcoind received the withdrawal unspent = l1.bitcoin.rpc.listunspent(0) @@ -526,21 +533,28 @@ def test_withdraw_misc(node_factory, bitcoind, chainparams): assert(withdrawal[0]['amount'] == Decimal('0.02')) + bitcoind.generate_block(1, wait_for_mempool=1) + sync_blockheight(bitcoind, [l1]) + # Now make sure two of them were marked as spent assert l1.db_query('SELECT COUNT(*) as c FROM outputs WHERE status=2')[0]['c'] == 2 + dont_spend_outputs(l1, out['txid']) + # Now send some money to l2. # lightningd uses P2SH-P2WPKH waddr = l2.rpc.newaddr('bech32')['bech32'] - l1.rpc.withdraw(waddr, 2 * amount) + out = l1.rpc.withdraw(waddr, amount) bitcoind.generate_block(1) # Make sure l2 received the withdrawal. wait_for(lambda: len(l2.rpc.listfunds()['outputs']) == 1) outputs = l2.db_query('SELECT value FROM outputs WHERE status=0;') - assert only_one(outputs)['value'] == 2 * amount + assert only_one(outputs)['value'] == amount # Now make sure an additional two of them were marked as spent + sync_blockheight(bitcoind, [l1]) + dont_spend_outputs(l1, out['txid']) assert l1.db_query('SELECT COUNT(*) as c FROM outputs WHERE status=2')[0]['c'] == 4 if chainparams['name'] != 'regtest': @@ -550,13 +564,16 @@ def test_withdraw_misc(node_factory, bitcoind, chainparams): # Address from: https://bc-2.jp/tools/bech32demo/index.html waddr = 'bcrt1qw508d6qejxtdg4y5r3zarvary0c5xw7kygt080' with pytest.raises(RpcError): - l1.rpc.withdraw('xx1qw508d6qejxtdg4y5r3zarvary0c5xw7kxpjzsx', 2 * amount) + l1.rpc.withdraw('xx1qw508d6qejxtdg4y5r3zarvary0c5xw7kxpjzsx', amount) with pytest.raises(RpcError): - l1.rpc.withdraw('tb1pw508d6qejxtdg4y5r3zarvary0c5xw7kdl9fad', 2 * amount) + l1.rpc.withdraw('tb1pw508d6qejxtdg4y5r3zarvary0c5xw7kdl9fad', amount) with pytest.raises(RpcError): - l1.rpc.withdraw('tb1qw508d6qejxtdg4y5r3zarvary0c5xw7kxxxxxx', 2 * amount) - l1.rpc.withdraw(waddr, 2 * amount) - bitcoind.generate_block(1) + l1.rpc.withdraw('tb1qw508d6qejxtdg4y5r3zarvary0c5xw7kxxxxxx', amount) + out = l1.rpc.withdraw(waddr, amount) + bitcoind.generate_block(1, wait_for_mempool=1) + sync_blockheight(bitcoind, [l1]) + dont_spend_outputs(l1, out['txid']) + # Now make sure additional two of them were marked as spent assert l1.db_query('SELECT COUNT(*) as c FROM outputs WHERE status=2')[0]['c'] == 6 @@ -564,44 +581,53 @@ def test_withdraw_misc(node_factory, bitcoind, chainparams): # Address from: https://bc-2.jp/tools/bech32demo/index.html waddr = 'bcrt1qrp33g0q5c5txsp9arysrx4k6zdkfs4nce4xj0gdcccefvpysxf3qzf4jry' with pytest.raises(RpcError): - l1.rpc.withdraw('xx1qrp33g0q5c5txsp9arysrx4k6zdkfs4nce4xj0gdcccefvpysxf3q0sl5k7', 2 * amount) + l1.rpc.withdraw('xx1qrp33g0q5c5txsp9arysrx4k6zdkfs4nce4xj0gdcccefvpysxf3q0sl5k7', amount) with pytest.raises(RpcError): - l1.rpc.withdraw('tb1prp33g0q5c5txsp9arysrx4k6zdkfs4nce4xj0gdcccefvpysxf3qsm03tq', 2 * amount) + l1.rpc.withdraw('tb1prp33g0q5c5txsp9arysrx4k6zdkfs4nce4xj0gdcccefvpysxf3qsm03tq', amount) with pytest.raises(RpcError): - l1.rpc.withdraw('tb1qrp33g0q5c5txsp9arysrx4k6zdkfs4nce4xj0gdcccefvpysxf3qxxxxxx', 2 * amount) - l1.rpc.withdraw(waddr, 2 * amount) - bitcoind.generate_block(1) + l1.rpc.withdraw('tb1qrp33g0q5c5txsp9arysrx4k6zdkfs4nce4xj0gdcccefvpysxf3qxxxxxx', amount) + out = l1.rpc.withdraw(waddr, amount) + bitcoind.generate_block(1, wait_for_mempool=1) + sync_blockheight(bitcoind, [l1]) + dont_spend_outputs(l1, out['txid']) # Now make sure additional two of them were marked as spent assert l1.db_query('SELECT COUNT(*) as c FROM outputs WHERE status=2')[0]['c'] == 8 # failure testing for invalid SegWit addresses, from BIP173 # HRP character out of range with pytest.raises(RpcError): - l1.rpc.withdraw(' 1nwldj5', 2 * amount) + l1.rpc.withdraw(' 1nwldj5', amount) # overall max length exceeded with pytest.raises(RpcError): - l1.rpc.withdraw('an84characterslonghumanreadablepartthatcontainsthenumber1andtheexcludedcharactersbio1569pvx', 2 * amount) + l1.rpc.withdraw('an84characterslonghumanreadablepartthatcontainsthenumber1andtheexcludedcharactersbio1569pvx', amount) # No separator character with pytest.raises(RpcError): - l1.rpc.withdraw('pzry9x0s0muk', 2 * amount) + l1.rpc.withdraw('pzry9x0s0muk', amount) # Empty HRP with pytest.raises(RpcError): - l1.rpc.withdraw('1pzry9x0s0muk', 2 * amount) + l1.rpc.withdraw('1pzry9x0s0muk', amount) # Invalid witness version with pytest.raises(RpcError): - l1.rpc.withdraw('BC13W508D6QEJXTDG4Y5R3ZARVARY0C5XW7KN40WF2', 2 * amount) + l1.rpc.withdraw('BC13W508D6QEJXTDG4Y5R3ZARVARY0C5XW7KN40WF2', amount) # Invalid program length for witness version 0 (per BIP141) with pytest.raises(RpcError): - l1.rpc.withdraw('BC1QR508D6QEJXTDG4Y5R3ZARVARYV98GJ9P', 2 * amount) + l1.rpc.withdraw('BC1QR508D6QEJXTDG4Y5R3ZARVARYV98GJ9P', amount) # Mixed case with pytest.raises(RpcError): - l1.rpc.withdraw('tb1qrp33g0q5c5txsp9arysrx4k6zdkfs4nce4xj0gdcccefvpysxf3q0sL5k7', 2 * amount) + l1.rpc.withdraw('tb1qrp33g0q5c5txsp9arysrx4k6zdkfs4nce4xj0gdcccefvpysxf3q0sL5k7', amount) # Non-zero padding in 8-to-5 conversion with pytest.raises(RpcError): - l1.rpc.withdraw('tb1qrp33g0q5c5txsp9arysrx4k6zdkfs4nce4xj0gdcccefvpysxf3pjxtptv', 2 * amount) + l1.rpc.withdraw('tb1qrp33g0q5c5txsp9arysrx4k6zdkfs4nce4xj0gdcccefvpysxf3pjxtptv', amount) - # Should have 6 outputs available. - assert l1.db_query('SELECT COUNT(*) as c FROM outputs WHERE status=0')[0]['c'] == 6 + # Should have 2 outputs available. + assert l1.db_query('SELECT COUNT(*) as c FROM outputs WHERE status=0')[0]['c'] == 2 + + # Unreserve everything. + inputs = [] + for out in l1.rpc.listfunds()['outputs']: + if out['reserved']: + inputs += [{'txid': out['txid'], 'vout': out['output']}] + l1.rpc.unreserveinputs(bitcoind.rpc.createpsbt(inputs, [])) # Test withdrawal to self. l1.rpc.withdraw(l1.rpc.newaddr('bech32')['bech32'], 'all', minconf=0) @@ -632,26 +658,34 @@ def test_withdraw_misc(node_factory, bitcoind, chainparams): {'type': 'chain_mvt', 'credit': 2000000000, 'debit': 0, 'tag': 'deposit'}, {'type': 'chain_mvt', 'credit': 0, 'debit': 0, 'tag': 'spend_track'}, {'type': 'chain_mvt', 'credit': 0, 'debit': 0, 'tag': 'spend_track'}, - {'type': 'chain_mvt', 'credit': 0, 'debit': 1993745000, 'tag': 'withdrawal'}, - {'type': 'chain_mvt', 'credit': 0, 'debit': 2000000000, 'tag': 'withdrawal'}, + [ + {'type': 'chain_mvt', 'credit': 0, 'debit': 2000000000, 'tag': 'withdrawal'}, + {'type': 'chain_mvt', 'credit': 0, 'debit': 1993745000, 'tag': 'withdrawal'}, + ], {'type': 'chain_mvt', 'credit': 0, 'debit': 6255000, 'tag': 'chain_fees'}, + {'type': 'chain_mvt', 'credit': 1993745000, 'debit': 0, 'tag': 'deposit'}, {'type': 'chain_mvt', 'credit': 0, 'debit': 0, 'tag': 'spend_track'}, {'type': 'chain_mvt', 'credit': 0, 'debit': 0, 'tag': 'spend_track'}, - {'type': 'chain_mvt', 'credit': 0, 'debit': 1993745000, 'tag': 'withdrawal'}, - {'type': 'chain_mvt', 'credit': 0, 'debit': 2000000000, 'tag': 'withdrawal'}, + [ + {'type': 'chain_mvt', 'credit': 0, 'debit': 2000000000, 'tag': 'withdrawal'}, + {'type': 'chain_mvt', 'credit': 0, 'debit': 1993745000, 'tag': 'withdrawal'}, + ], {'type': 'chain_mvt', 'credit': 0, 'debit': 6255000, 'tag': 'chain_fees'}, {'type': 'chain_mvt', 'credit': 1993745000, 'debit': 0, 'tag': 'deposit'}, - {'type': 'chain_mvt', 'credit': 1993745000, 'debit': 0, 'tag': 'deposit'}, {'type': 'chain_mvt', 'credit': 0, 'debit': 0, 'tag': 'spend_track'}, {'type': 'chain_mvt', 'credit': 0, 'debit': 0, 'tag': 'spend_track'}, - {'type': 'chain_mvt', 'credit': 0, 'debit': 1993745000, 'tag': 'withdrawal'}, - {'type': 'chain_mvt', 'credit': 0, 'debit': 2000000000, 'tag': 'withdrawal'}, + [ + {'type': 'chain_mvt', 'credit': 0, 'debit': 2000000000, 'tag': 'withdrawal'}, + {'type': 'chain_mvt', 'credit': 0, 'debit': 1993745000, 'tag': 'withdrawal'}, + ], {'type': 'chain_mvt', 'credit': 0, 'debit': 6255000, 'tag': 'chain_fees'}, {'type': 'chain_mvt', 'credit': 1993745000, 'debit': 0, 'tag': 'deposit'}, {'type': 'chain_mvt', 'credit': 0, 'debit': 0, 'tag': 'spend_track'}, {'type': 'chain_mvt', 'credit': 0, 'debit': 0, 'tag': 'spend_track'}, - {'type': 'chain_mvt', 'credit': 0, 'debit': 1993385000, 'tag': 'withdrawal'}, - {'type': 'chain_mvt', 'credit': 0, 'debit': 2000000000, 'tag': 'withdrawal'}, + [ + {'type': 'chain_mvt', 'credit': 0, 'debit': 1993385000, 'tag': 'withdrawal'}, + {'type': 'chain_mvt', 'credit': 0, 'debit': 2000000000, 'tag': 'withdrawal'}, + ], {'type': 'chain_mvt', 'credit': 0, 'debit': 6615000, 'tag': 'chain_fees'}, {'type': 'chain_mvt', 'credit': 1993385000, 'debit': 0, 'tag': 'deposit'}, {'type': 'chain_mvt', 'credit': 0, 'debit': 0, 'tag': 'spend_track'}, diff --git a/tests/test_plugin.py b/tests/test_plugin.py index 518f70311ba3..c9eb8efbbdea 100644 --- a/tests/test_plugin.py +++ b/tests/test_plugin.py @@ -1509,8 +1509,11 @@ def test_coin_movement_notices(node_factory, bitcoind, chainparams): l2_wallet_mvts = [ {'type': 'chain_mvt', 'credit': 2000000000, 'debit': 0, 'tag': 'deposit'}, {'type': 'chain_mvt', 'credit': 0, 'debit': 0, 'tag': 'spend_track'}, - {'type': 'chain_mvt', 'credit': 0, 'debit': 995425000, 'tag': 'withdrawal'}, - {'type': 'chain_mvt', 'credit': 0, 'debit': 1000000000, 'tag': 'withdrawal'}, + # Could go in either order + [ + {'type': 'chain_mvt', 'credit': 0, 'debit': 995425000, 'tag': 'withdrawal'}, + {'type': 'chain_mvt', 'credit': 0, 'debit': 1000000000, 'tag': 'withdrawal'}, + ], {'type': 'chain_mvt', 'credit': 0, 'debit': 4575000, 'tag': 'chain_fees'}, {'type': 'chain_mvt', 'credit': 995425000, 'debit': 0, 'tag': 'deposit'}, {'type': 'chain_mvt', 'credit': 100001000, 'debit': 0, 'tag': 'deposit'}, @@ -1528,8 +1531,11 @@ def test_coin_movement_notices(node_factory, bitcoind, chainparams): l2_wallet_mvts = [ {'type': 'chain_mvt', 'credit': 2000000000, 'debit': 0, 'tag': 'deposit'}, {'type': 'chain_mvt', 'credit': 0, 'debit': 0, 'tag': 'spend_track'}, - {'type': 'chain_mvt', 'credit': 0, 'debit': 995425000, 'tag': 'withdrawal'}, - {'type': 'chain_mvt', 'credit': 0, 'debit': 1000000000, 'tag': 'withdrawal'}, + # Could go in either order + [ + {'type': 'chain_mvt', 'credit': 0, 'debit': 995425000, 'tag': 'withdrawal'}, + {'type': 'chain_mvt', 'credit': 0, 'debit': 1000000000, 'tag': 'withdrawal'}, + ], {'type': 'chain_mvt', 'credit': 0, 'debit': 4575000, 'tag': 'chain_fees'}, {'type': 'chain_mvt', 'credit': 995425000, 'debit': 0, 'tag': 'deposit'}, {'type': 'chain_mvt', 'credit': 100001000, 'debit': 0, 'tag': 'deposit'}, From 7a93363b0b0b1613afc089238d8fe64acbf4d95a Mon Sep 17 00:00:00 2001 From: Rusty Russell Date: Fri, 28 Aug 2020 12:15:57 +0930 Subject: [PATCH 20/30] pytest: remove duplicate tests. These tests appear in both test_misc.py and test_wallet.py. I checked, and they're the only duplcates; they were moved in 0543149b89086b9367465d69e5e3248ba2a90886 then re-added (merge error?) in 67fc8ffbcff7cd0d677c496ee5c800860ceca3c5. Signed-off-by: Rusty Russell --- tests/test_misc.py | 55 ---------------------------------------------- 1 file changed, 55 deletions(-) diff --git a/tests/test_misc.py b/tests/test_misc.py index 8caaa5c326b6..b910567b14f7 100644 --- a/tests/test_misc.py +++ b/tests/test_misc.py @@ -704,61 +704,6 @@ def dont_spend_outputs(n, txid): check_coin_moves(l1, 'wallet', wallet_moves, chainparams) -def test_minconf_withdraw(node_factory, bitcoind): - """Issue 2518: ensure that ridiculous confirmation levels don't overflow - - The number of confirmations is used to compute a maximum height that is to - be accepted. If the current height is smaller than the number of - confirmations we wrap around and just select everything. The fix is to - clamp the maxheight parameter to a positive small number. - - """ - amount = 1000000 - # Don't get any funds from previous runs. - l1 = node_factory.get_node(random_hsm=True) - addr = l1.rpc.newaddr()['bech32'] - - # Add some funds to withdraw later - for i in range(10): - l1.bitcoin.rpc.sendtoaddress(addr, amount / 10**8 + 0.01) - - bitcoind.generate_block(1) - - wait_for(lambda: len(l1.rpc.listfunds()['outputs']) == 10) - with pytest.raises(RpcError): - l1.rpc.withdraw(destination=addr, satoshi=10000, feerate='normal', minconf=9999999) - - -def test_addfunds_from_block(node_factory, bitcoind): - """Send funds to the daemon without telling it explicitly - """ - # Previous runs with same bitcoind can leave funds! - l1 = node_factory.get_node(random_hsm=True) - - addr = l1.rpc.newaddr()['bech32'] - bitcoind.rpc.sendtoaddress(addr, 0.1) - bitcoind.generate_block(1) - - wait_for(lambda: len(l1.rpc.listfunds()['outputs']) == 1) - - outputs = l1.db_query('SELECT value FROM outputs WHERE status=0;') - assert only_one(outputs)['value'] == 10000000 - - # The address we detect must match what was paid to. - output = only_one(l1.rpc.listfunds()['outputs']) - assert output['address'] == addr - - # Send all our money to a P2WPKH address this time. - addr = l1.rpc.newaddr("bech32")['bech32'] - l1.rpc.withdraw(addr, "all") - bitcoind.generate_block(1) - time.sleep(1) - - # The address we detect must match what was paid to. - output = only_one(l1.rpc.listfunds()['outputs']) - assert output['address'] == addr - - def test_io_logging(node_factory, executor): l1 = node_factory.get_node(options={'log-level': 'io'}) l2 = node_factory.get_node() From 83298c030a7ecd759900149c0dcda980be542c5e Mon Sep 17 00:00:00 2001 From: Rusty Russell Date: Fri, 28 Aug 2020 13:26:32 +0930 Subject: [PATCH 21/30] wallet: switch over to withdraw in module, remove lots of unused code. This removes the reservation cleanup at startup, too, now they're all using 'reserved_til'. This changes test_withdraw, since it asserted that outputs were marked spent as soon as we broadcast a transaction: now they're reserved until it's mined. Similarly, test_addfunds_from_block assumed we'd see funds as soon as we broadcast the tx. Signed-off-by: Rusty Russell Changelog-Changed: JSON-RPC: `withdraw` now randomizes input and output order, not BIP69. --- common/Makefile | 4 +- common/wallet_tx.c | 264 --------------------------------- common/wallet_tx.h | 61 -------- common/withdraw_tx.c | 37 ----- common/withdraw_tx.h | 34 ----- hsmd/Makefile | 3 +- hsmd/hsmd.c | 1 - lightningd/Makefile | 2 - lightningd/channel_control.c | 1 - lightningd/channel_control.h | 1 + lightningd/lightningd.c | 4 - lightningd/opening_control.c | 1 - plugins/txprepare.c | 2 +- tests/test_misc.py | 4 +- tests/test_wallet.py | 19 ++- wallet/reservation.c | 16 +- wallet/test/run-wallet.c | 39 +++-- wallet/wallet.c | 265 +-------------------------------- wallet/wallet.h | 40 ----- wallet/walletrpc.c | 279 ----------------------------------- 20 files changed, 65 insertions(+), 1012 deletions(-) delete mode 100644 common/wallet_tx.c delete mode 100644 common/wallet_tx.h delete mode 100644 common/withdraw_tx.c delete mode 100644 common/withdraw_tx.h diff --git a/common/Makefile b/common/Makefile index c407214e0311..b62e738c08f0 100644 --- a/common/Makefile +++ b/common/Makefile @@ -74,10 +74,8 @@ COMMON_SRC_NOGEN := \ common/utxo.c \ common/version.c \ common/wallet.c \ - common/wallet_tx.c \ common/wireaddr.c \ - common/wire_error.c \ - common/withdraw_tx.c + common/wire_error.c COMMON_SRC_GEN := common/status_wiregen.c common/peer_status_wiregen.c diff --git a/common/wallet_tx.c b/common/wallet_tx.c deleted file mode 100644 index da9a1a171523..000000000000 --- a/common/wallet_tx.c +++ /dev/null @@ -1,264 +0,0 @@ -#include -#include -#include -#include -#include -#include -#include -#include -#include - -void wtx_init(struct command *cmd, struct wallet_tx *wtx, struct amount_sat max) -{ - wtx->cmd = cmd; - wtx->amount = max; -} - -struct command_result *param_wtx(struct command *cmd, - const char *name, - const char *buffer, - const jsmntok_t *tok, - struct wallet_tx *wtx) -{ - struct amount_sat max = wtx->amount; - - if (json_tok_streq(buffer, tok, "all")) { - wtx->all_funds = true; - return NULL; - } - wtx->all_funds = false; - - if (!parse_amount_sat(&wtx->amount, - buffer + tok->start, tok->end - tok->start)) - return command_fail(cmd, JSONRPC2_INVALID_PARAMS, - "'%s' should be an amount in satoshis or all, not '%.*s'", - name, - tok->end - tok->start, - buffer + tok->start); - - if (amount_sat_greater(wtx->amount, max)) - return command_fail(wtx->cmd, FUND_MAX_EXCEEDED, - "Amount exceeded %s", - type_to_string(tmpctx, struct amount_sat, - &max)); - return NULL; -} - -struct command_result *param_utxos(struct command *cmd, - const char *name, - const char *buffer, - const jsmntok_t *tok, - const struct utxo ***utxos) -{ - size_t i; - const jsmntok_t *curr; - struct bitcoin_txid **txids = tal_arr(cmd, struct bitcoin_txid*, 0); - unsigned int **outnums = tal_arr(cmd, unsigned int*, 0); - - json_for_each_arr(i, curr, tok) { - jsmntok_t txid_tok, outnum_tok; - if (!split_tok(buffer, curr, ':', &txid_tok, &outnum_tok)) - return command_fail(cmd, JSONRPC2_INVALID_PARAMS, - "Could not decode the outpoint from \"%s\"" - " The utxos should be specified as" - " 'txid:output_index'.", - json_strdup(tmpctx, buffer, curr)); - - struct bitcoin_txid *txid = tal(txids, struct bitcoin_txid); - unsigned int *outnum = tal(txids, unsigned int); - if (!json_to_txid(buffer, (const jsmntok_t*)&txid_tok, txid)) { - return command_fail(cmd, JSONRPC2_INVALID_PARAMS, - "Could not get a txid out of \"%s\"", - json_strdup(tmpctx, buffer, &txid_tok)); - } - if (!json_to_number(buffer, (const jsmntok_t*)&outnum_tok, outnum)) - return command_fail(cmd, JSONRPC2_INVALID_PARAMS, - "Could not get a vout out of \"%s\"", - json_strdup(tmpctx, buffer, &outnum_tok)); - - tal_arr_expand(&txids, txid); - tal_arr_expand(&outnums, outnum); - } - - if (!tal_count(txids) || !tal_count(outnums)) - return command_fail(cmd, JSONRPC2_INVALID_PARAMS, - "Please specify an array of 'txid:output_index'," - " not \"%.*s\"", - tok->end - tok->start, - buffer + tok->start); - - *utxos = wallet_select_specific(cmd, cmd->ld->wallet, txids, outnums); - tal_free(txids); - tal_free(outnums); - - if (!*utxos) - return command_fail(cmd, JSONRPC2_INVALID_PARAMS, - "Could not decode all of the outpoints. The utxos" - " should be specified as an array of " - " 'txid:output_index'."); - if (tal_count(*utxos) == 0) - return command_fail(cmd, JSONRPC2_INVALID_PARAMS, - "No matching utxo was found from the wallet. " - "You can get a list of the wallet utxos with" - " the `listfunds` RPC call."); - return NULL; -} - -static struct command_result *check_amount(const struct wallet_tx *wtx, - struct amount_sat amount) -{ - if (tal_count(wtx->utxos) == 0) { - /* Since it's possible the lack of utxos is because we haven't finished - * syncing yet, report a sync timing error first */ - if (!topology_synced(wtx->cmd->ld->topology)) - return command_fail(wtx->cmd, FUNDING_STILL_SYNCING_BITCOIN, - "Still syncing with bitcoin network"); - - return command_fail(wtx->cmd, FUND_CANNOT_AFFORD, - "Cannot afford transaction"); - } - - if (amount_sat_less(amount, chainparams->dust_limit)) { - return command_fail(wtx->cmd, FUND_OUTPUT_IS_DUST, - "Output %s would be dust", - type_to_string(tmpctx, struct amount_sat, - &amount)); - } - return NULL; -} - -struct command_result *wtx_select_utxos(struct wallet_tx *tx, - u32 fee_rate_per_kw, - size_t out_len, - u32 maxheight) -{ - struct command_result *res; - struct amount_sat fee_estimate; - - if (tx->all_funds) { - struct amount_sat amount; - tx->utxos = wallet_select_all(tx, tx->cmd->ld->wallet, - fee_rate_per_kw, out_len, - maxheight, - &amount, - &fee_estimate); - res = check_amount(tx, amount); - if (res) - return res; - - /* tx->amount is max permissible */ - if (amount_sat_less_eq(amount, tx->amount)) { - tx->change = AMOUNT_SAT(0); - tx->change_key_index = 0; - tx->amount = amount; - return NULL; - } - - /* Too much? Try again, but ask for limit instead. */ - tx->all_funds = false; - tx->utxos = tal_free(tx->utxos); - } - - tx->utxos = wallet_select_coins(tx, tx->cmd->ld->wallet, - true, tx->amount, - fee_rate_per_kw, out_len, - maxheight, - &fee_estimate, &tx->change); - if (!tx->utxos) { - /* Try again, without change this time */ - tx->utxos = wallet_select_coins(tx, tx->cmd->ld->wallet, - false, tx->amount, - fee_rate_per_kw, out_len, - maxheight, - &fee_estimate, &tx->change); - - } - - res = check_amount(tx, tx->amount); - if (res) - return res; - - if (amount_sat_less(tx->change, chainparams->dust_limit)) { - tx->change = AMOUNT_SAT(0); - tx->change_key_index = 0; - } else { - tx->change_key_index = wallet_get_newindex(tx->cmd->ld); - } - return NULL; -} - -struct command_result *wtx_from_utxos(struct wallet_tx *tx, - u32 fee_rate_per_kw, - size_t out_len, - u32 maxheight, - const struct utxo **utxos) -{ - size_t weight; - struct amount_sat total_amount, fee_estimate; - - tx->change = AMOUNT_SAT(0); - tx->change_key_index = 0; - total_amount = AMOUNT_SAT(0); - - /* The transaction has `tal_count(tx.utxos)` inputs and one output */ - /* (version + in count + out count + locktime) (index + value + script length) */ - /* + segwit marker + flag */ - weight = 4 * (4 + 1 + 1 + 4) + 4 * (8 + 1 + out_len) + 1 + 1; - for (size_t i = 0; i < tal_count(utxos); i++) { - if (maxheight > 0 && - (!utxos[i]->blockheight || *utxos[i]->blockheight > maxheight)) { - tal_arr_remove(&utxos, i); - continue; - } - /* txid + index + sequence + script_len */ - weight += (32 + 4 + 4 + 1) * 4; - /* P2SH variants include push of <0 <20-byte-key-hash>> */ - if (utxos[i]->is_p2sh) - weight += 23 * 4; - /* Account for witness (1 byte count + sig + key) */ - weight += 1 + (1 + 73 + 1 + 33); - if (!amount_sat_add(&total_amount, total_amount, utxos[i]->amount)) - fatal("Overflow when computing input amount"); - } - tx->utxos = tal_steal(tx, utxos); - - if (!tx->all_funds && amount_sat_less(tx->amount, total_amount) - && !amount_sat_sub(&tx->change, total_amount, tx->amount)) - fatal("Overflow when computing change"); - - if (amount_sat_greater_eq(tx->change, chainparams->dust_limit)) { - /* Add the change output's weight */ - weight += (8 + 1 + out_len) * 4; - } - - fee_estimate = amount_tx_fee(fee_rate_per_kw, weight); - - if (tx->all_funds || amount_sat_eq(tx->change, AMOUNT_SAT(0))) { - tx->amount = total_amount; - if (!amount_sat_sub(&tx->amount, tx->amount, fee_estimate)) - return command_fail(tx->cmd, FUND_CANNOT_AFFORD, - "Cannot afford transaction with %s" - " sats of fees, make sure to use " - "confirmed utxos.", - type_to_string(tmpctx, struct amount_sat, - &fee_estimate)); - } else { - if (!amount_sat_sub(&tx->change, tx->change, fee_estimate)) { - /* Try again without a change output */ - weight -= (8 + 1 + out_len) * 4; - fee_estimate = amount_tx_fee(fee_rate_per_kw, weight); - if (!amount_sat_sub(&tx->change, tx->change, fee_estimate)) - return command_fail(tx->cmd, FUND_CANNOT_AFFORD, - "Cannot afford transaction with %s" - " sats of fees, make sure to use " - "confirmed utxos.", - type_to_string(tmpctx, struct amount_sat, - &fee_estimate)); - tx->change = AMOUNT_SAT(0); - } else { - tx->change_key_index = wallet_get_newindex(tx->cmd->ld); - } - } - - return check_amount(tx, tx->amount); -} diff --git a/common/wallet_tx.h b/common/wallet_tx.h deleted file mode 100644 index 1b8e95229d48..000000000000 --- a/common/wallet_tx.h +++ /dev/null @@ -1,61 +0,0 @@ -#ifndef LIGHTNING_COMMON_WALLET_TX_H -#define LIGHTNING_COMMON_WALLET_TX_H -#include "config.h" -#include -#include -#include -#include -#include - -/* A specification of funds in the wallet used for funding channels and - * withdrawal. - */ -struct wallet_tx { - struct command *cmd; - struct amount_sat amount, change; - u32 change_key_index; - const struct utxo **utxos; - - bool all_funds; /* In this case, amount is a maximum. */ -}; - -void wtx_init(struct command *cmd, struct wallet_tx *wtx, struct amount_sat max); - -struct command_result *param_wtx(struct command *cmd, - const char *name, - const char *buffer, - const jsmntok_t *tok, - struct wallet_tx *wtx); - -struct command_result *param_utxos(struct command *cmd, - const char *name, - const char *buffer, - const jsmntok_t *tok, - const struct utxo ***utxos); - -struct command_result *wtx_select_utxos(struct wallet_tx *tx, - u32 fee_rate_per_kw, - size_t out_len, - u32 maxheight); - -struct command_result *wtx_from_utxos(struct wallet_tx *tx, - u32 fee_rate_per_kw, - size_t out_len, - u32 maxheight, - const struct utxo **utxos); - -static inline u32 minconf_to_maxheight(u32 minconf, struct lightningd *ld) -{ - /* No confirmations is special, we need to disable the check in the - * selection */ - if (minconf == 0) - return 0; - - /* Avoid wrapping around and suddenly allowing any confirmed - * outputs. Since we can't have a coinbase output, and 0 is taken for - * the disable case, we can just clamp to 1. */ - if (minconf >= ld->topology->tip->height) - return 1; - return ld->topology->tip->height - minconf + 1; -} -#endif /* LIGHTNING_COMMON_WALLET_TX_H */ diff --git a/common/withdraw_tx.c b/common/withdraw_tx.c deleted file mode 100644 index b52aac517c84..000000000000 --- a/common/withdraw_tx.c +++ /dev/null @@ -1,37 +0,0 @@ -#include "withdraw_tx.h" -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -struct bitcoin_tx *withdraw_tx(const tal_t *ctx, - const struct chainparams *chainparams, - const struct utxo **utxos, - struct bitcoin_tx_output **outputs, - const struct ext_key *bip32_base, - u32 nlocktime) -{ - struct bitcoin_tx *tx; - int output_count; - - tx = tx_spending_utxos(ctx, chainparams, utxos, bip32_base, - false, tal_count(outputs), nlocktime, - BITCOIN_TX_DEFAULT_SEQUENCE - 1); - - output_count = bitcoin_tx_add_multi_outputs(tx, outputs); - assert(output_count == tal_count(outputs)); - - permute_outputs(tx, NULL, (const void **)outputs); - permute_inputs(tx, (const void **)utxos); - - bitcoin_tx_finalize(tx); - assert(bitcoin_tx_check(tx)); - return tx; -} - diff --git a/common/withdraw_tx.h b/common/withdraw_tx.h deleted file mode 100644 index cf5d67af18c6..000000000000 --- a/common/withdraw_tx.h +++ /dev/null @@ -1,34 +0,0 @@ -#ifndef LIGHTNING_COMMON_WITHDRAW_TX_H -#define LIGHTNING_COMMON_WITHDRAW_TX_H -#include "config.h" -#include -#include -#include -#include -#include - -struct bitcoin_tx; -struct ext_key; -struct privkey; -struct pubkey; -struct bitcoin_address; -struct utxo; - -/** - * withdraw_tx - Create a p2pkh withdrawal transaction - * - * @ctx: context to tal from. - * @chainparams: (in) the params for the created transaction. - * @utxos: (in/out) tal_arr of UTXO pointers to spend (permuted to match) - * @outputs: (in) tal_arr of bitcoin_tx_output, scriptPubKeys with amount to send to. - * @bip32_base: (in) bip32 base for key derivation, or NULL. - * @nlocktime: (in) the value to set as the transaction's nLockTime. - */ -struct bitcoin_tx *withdraw_tx(const tal_t *ctx, - const struct chainparams *chainparams, - const struct utxo **utxos, - struct bitcoin_tx_output **outputs, - const struct ext_key *bip32_base, - u32 nlocktime); - -#endif /* LIGHTNING_COMMON_WITHDRAW_TX_H */ diff --git a/hsmd/Makefile b/hsmd/Makefile index 2b5c0b291fef..f3ad5ca72fd4 100644 --- a/hsmd/Makefile +++ b/hsmd/Makefile @@ -39,8 +39,7 @@ HSMD_COMMON_OBJS := \ common/type_to_string.o \ common/utils.o \ common/utxo.o \ - common/version.o \ - common/withdraw_tx.o + common/version.o lightningd/lightning_hsmd: $(HSMD_OBJS) $(HSMD_COMMON_OBJS) $(BITCOIN_OBJS) $(WIRE_OBJS) diff --git a/hsmd/hsmd.c b/hsmd/hsmd.c index f4bd92a903a8..1bd52a4dfb04 100644 --- a/hsmd/hsmd.c +++ b/hsmd/hsmd.c @@ -37,7 +37,6 @@ #include #include #include -#include #include #include #include diff --git a/lightningd/Makefile b/lightningd/Makefile index b3190950afd9..2894fbedca3c 100644 --- a/lightningd/Makefile +++ b/lightningd/Makefile @@ -110,10 +110,8 @@ LIGHTNINGD_COMMON_OBJS := \ common/utxo.o \ common/version.o \ common/wallet.o \ - common/wallet_tx.o \ common/wire_error.o \ common/wireaddr.o \ - common/withdraw_tx.o include wallet/Makefile diff --git a/lightningd/channel_control.c b/lightningd/channel_control.c index fa5488eb358f..82173e99617f 100644 --- a/lightningd/channel_control.c +++ b/lightningd/channel_control.c @@ -12,7 +12,6 @@ #include #include #include -#include #include #include #include diff --git a/lightningd/channel_control.h b/lightningd/channel_control.h index cf234a5e773a..016b2e6d1c65 100644 --- a/lightningd/channel_control.h +++ b/lightningd/channel_control.h @@ -8,6 +8,7 @@ struct channel; struct crypto_state; struct lightningd; struct per_peer_state; +struct peer; void peer_start_channeld(struct channel *channel, struct per_peer_state *pps, diff --git a/lightningd/lightningd.c b/lightningd/lightningd.c index 8928e94639c0..bc8cb2efc5c4 100644 --- a/lightningd/lightningd.c +++ b/lightningd/lightningd.c @@ -935,9 +935,6 @@ int main(int argc, char *argv[]) min_blockheight, max_blockheight); db_begin_transaction(ld->wallet->db); - /*~ Tell the wallet to start figuring out what to do for any reserved - * unspent outputs we may have crashed with. */ - wallet_clean_utxos(ld->wallet, ld->topology->bitcoind); /*~ Pull peers, channels and HTLCs from db. Needs to happen after the * topology is initialized since some decisions rely on being able to @@ -1045,7 +1042,6 @@ int main(int argc, char *argv[]) * unreserving UTXOs (see #1737) */ db_begin_transaction(ld->wallet->db); tal_free(ld->jsonrpc); - free_unreleased_txs(ld->wallet); db_commit_transaction(ld->wallet->db); /* Clean our our HTLC maps, since they use malloc. */ diff --git a/lightningd/opening_control.c b/lightningd/opening_control.c index 0d96a9b8e15a..023f567e77e1 100644 --- a/lightningd/opening_control.c +++ b/lightningd/opening_control.c @@ -15,7 +15,6 @@ #include #include #include -#include #include #include #include diff --git a/plugins/txprepare.c b/plugins/txprepare.c index bf587e113465..7ce05cb48afd 100644 --- a/plugins/txprepare.c +++ b/plugins/txprepare.c @@ -521,7 +521,7 @@ static const struct plugin_command commands[] = { json_txsend }, { - "newwithdraw", + "withdraw", "bitcoin", "Send funds to {destination} address", "Send to {destination} {satoshi} (or 'all') at optional {feerate} using utxos from {minconf} or {utxos}.", diff --git a/tests/test_misc.py b/tests/test_misc.py index b910567b14f7..491dd6fead07 100644 --- a/tests/test_misc.py +++ b/tests/test_misc.py @@ -522,7 +522,7 @@ def dont_spend_outputs(n, txid): l1.rpc.withdraw(waddr, 'not an amount') with pytest.raises(RpcError): l1.rpc.withdraw(waddr, -amount) - with pytest.raises(RpcError, match=r'Cannot afford transaction'): + with pytest.raises(RpcError, match=r'Could not afford'): l1.rpc.withdraw(waddr, amount * 100) out = l1.rpc.withdraw(waddr, amount) @@ -638,7 +638,7 @@ def dont_spend_outputs(n, txid): assert l1.db_query('SELECT COUNT(*) as c FROM outputs WHERE status=0')[0]['c'] == 0 # This should fail, can't even afford fee. - with pytest.raises(RpcError, match=r'Cannot afford transaction'): + with pytest.raises(RpcError, match=r'Could not afford'): l1.rpc.withdraw(waddr, 'all') bitcoind.generate_block(1) diff --git a/tests/test_wallet.py b/tests/test_wallet.py index 53e94797071e..ed7fc6400da1 100644 --- a/tests/test_wallet.py +++ b/tests/test_wallet.py @@ -12,7 +12,6 @@ import os import pytest import subprocess -import time import unittest @@ -42,7 +41,7 @@ def test_withdraw(node_factory, bitcoind): l1.rpc.withdraw(waddr, 'not an amount') with pytest.raises(RpcError): l1.rpc.withdraw(waddr, -amount) - with pytest.raises(RpcError, match=r'Cannot afford transaction'): + with pytest.raises(RpcError, match=r'Could not afford'): l1.rpc.withdraw(waddr, amount * 100) out = l1.rpc.withdraw(waddr, 2 * amount) @@ -67,15 +66,23 @@ def test_withdraw(node_factory, bitcoind): # lightningd uses P2SH-P2WPKH waddr = l2.rpc.newaddr('bech32')['bech32'] l1.rpc.withdraw(waddr, 2 * amount) + + # Now make sure an additional two of them were marked as reserved + assert l1.db_query('SELECT COUNT(*) as c FROM outputs WHERE status=2')[0]['c'] == 2 + assert l1.db_query('SELECT COUNT(*) as c FROM outputs WHERE status=1')[0]['c'] == 2 + + # They're turned into spent once the node sees them mined. bitcoind.generate_block(1) + sync_blockheight(l1.bitcoin, [l1, l2]) # Make sure l2 received the withdrawal. - wait_for(lambda: len(l2.rpc.listfunds()['outputs']) == 1) + assert len(l2.rpc.listfunds()['outputs']) == 1 outputs = l2.db_query('SELECT value FROM outputs WHERE status=0;') assert only_one(outputs)['value'] == 2 * amount # Now make sure an additional two of them were marked as spent assert l1.db_query('SELECT COUNT(*) as c FROM outputs WHERE status=2')[0]['c'] == 4 + assert l1.db_query('SELECT COUNT(*) as c FROM outputs WHERE status=1')[0]['c'] == 0 # Simple test for withdrawal to P2WPKH # Address from: https://bc-2.jp/tools/bech32demo/index.html @@ -88,6 +95,7 @@ def test_withdraw(node_factory, bitcoind): l1.rpc.withdraw('tb1qw508d6qejxtdg4y5r3zarvary0c5xw7kxxxxxx', 2 * amount) l1.rpc.withdraw(waddr, 2 * amount) bitcoind.generate_block(1) + sync_blockheight(l1.bitcoin, [l1]) # Now make sure additional two of them were marked as spent assert l1.db_query('SELECT COUNT(*) as c FROM outputs WHERE status=2')[0]['c'] == 6 @@ -102,6 +110,7 @@ def test_withdraw(node_factory, bitcoind): l1.rpc.withdraw('tb1qrp33g0q5c5txsp9arysrx4k6zdkfs4nce4xj0gdcccefvpysxf3qxxxxxx', 2 * amount) l1.rpc.withdraw(waddr, 2 * amount) bitcoind.generate_block(1) + sync_blockheight(l1.bitcoin, [l1]) # Now make sure additional two of them were marked as spent assert l1.db_query('SELECT COUNT(*) as c FROM outputs WHERE status=2')[0]['c'] == 8 @@ -143,7 +152,7 @@ def test_withdraw(node_factory, bitcoind): assert l1.db_query('SELECT COUNT(*) as c FROM outputs WHERE status=0')[0]['c'] == 0 # This should fail, can't even afford fee. - with pytest.raises(RpcError, match=r'Cannot afford transaction'): + with pytest.raises(RpcError, match=r'Could not afford'): l1.rpc.withdraw(waddr, 'all') # Add some funds to withdraw @@ -225,7 +234,7 @@ def test_addfunds_from_block(node_factory, bitcoind): addr = l1.rpc.newaddr("bech32")['bech32'] l1.rpc.withdraw(addr, "all") bitcoind.generate_block(1) - time.sleep(1) + sync_blockheight(bitcoind, [l1]) # The address we detect must match what was paid to. output = only_one(l1.rpc.listfunds()['outputs']) diff --git a/wallet/reservation.c b/wallet/reservation.c index 4d2474434b0f..aa20177bf150 100644 --- a/wallet/reservation.c +++ b/wallet/reservation.c @@ -6,7 +6,6 @@ #include #include #include -#include #include #include #include @@ -314,6 +313,21 @@ static struct command_result *finish_psbt(struct command *cmd, return command_success(cmd, response); } +static inline u32 minconf_to_maxheight(u32 minconf, struct lightningd *ld) +{ + /* No confirmations is special, we need to disable the check in the + * selection */ + if (minconf == 0) + return 0; + + /* Avoid wrapping around and suddenly allowing any confirmed + * outputs. Since we can't have a coinbase output, and 0 is taken for + * the disable case, we can just clamp to 1. */ + if (minconf >= ld->topology->tip->height) + return 1; + return ld->topology->tip->height - minconf + 1; +} + static struct command_result *json_fundpsbt(struct command *cmd, const char *buffer, const jsmntok_t *obj UNNEEDED, diff --git a/wallet/test/run-wallet.c b/wallet/test/run-wallet.c index 427cda0eda60..652da3ea6631 100644 --- a/wallet/test/run-wallet.c +++ b/wallet/test/run-wallet.c @@ -908,7 +908,7 @@ static bool test_wallet_outputs(struct lightningd *ld, const tal_t *ctx) struct utxo u; struct pubkey pk; struct node_id id; - struct amount_sat fee_estimate, change_satoshis; + struct utxo *one_utxo; const struct utxo **utxos; CHECK(w); @@ -941,19 +941,23 @@ static bool test_wallet_outputs(struct lightningd *ld, const tal_t *ctx) "wallet_add_utxo with close_info"); /* Now select them */ - utxos = wallet_select_coins(w, w, true, AMOUNT_SAT(2), 0, 21, - 0 /* no confirmations required */, - &fee_estimate, &change_satoshis); - CHECK(utxos && tal_count(utxos) == 2); + utxos = tal_arr(w, const struct utxo *, 0); + while ((one_utxo = wallet_find_utxo(w, w, 0, NULL, 253, + 0 /* no confirmations required */, + utxos)) != NULL) { + tal_arr_expand(&utxos, one_utxo); + } + CHECK(tal_count(utxos) == 2); + + if (utxos[0]->close_info) + u = *utxos[0]; + else + u = *utxos[1]; - u = *utxos[1]; CHECK(u.close_info->channel_id == 42 && pubkey_eq(u.close_info->commitment_point, &pk) && node_id_eq(&u.close_info->peer_id, &id) && u.close_info->option_anchor_outputs == false); - /* Now un-reserve them for the tests below */ - tal_free(utxos); - /* Attempt to reserve the utxo */ CHECK_MSG(wallet_update_output_status(w, &u.txid, u.outnum, @@ -993,12 +997,19 @@ static bool test_wallet_outputs(struct lightningd *ld, const tal_t *ctx) "wallet_add_utxo with close_info no commitment_point"); /* Now select it */ - utxos = wallet_select_coins(w, w, true, AMOUNT_SAT(5), 0, 21, - 0 /* no confirmations required */, - &fee_estimate, &change_satoshis); - CHECK(utxos && tal_count(utxos) == 2); + utxos = tal_arr(w, const struct utxo *, 0); + while ((one_utxo = wallet_find_utxo(w, w, 0, NULL, 253, + 0 /* no confirmations required */, + utxos)) != NULL) { + tal_arr_expand(&utxos, one_utxo); + } + CHECK(tal_count(utxos) == 2); + + if (utxos[0]->close_info) + u = *utxos[0]; + else + u = *utxos[1]; - u = *utxos[1]; CHECK(u.close_info->channel_id == 42 && u.close_info->commitment_point == NULL && node_id_eq(&u.close_info->peer_id, &id) && diff --git a/wallet/wallet.c b/wallet/wallet.c index dec065ff35cc..85b74a64f996 100644 --- a/wallet/wallet.c +++ b/wallet/wallet.c @@ -67,7 +67,6 @@ struct wallet *wallet_new(struct lightningd *ld, struct timers *timers, wallet->bip32_base = tal_steal(wallet, bip32_base); wallet->keyscan_gap = 50; list_head_init(&wallet->unstored_payments); - list_head_init(&wallet->unreleased_txs); wallet->db = db_setup(wallet, ld, wallet->bip32_base); db_begin_transaction(wallet->db); @@ -162,7 +161,7 @@ static bool wallet_add_utxo(struct wallet *w, struct utxo *utxo, static struct utxo *wallet_stmt2output(const tal_t *ctx, struct db_stmt *stmt) { struct utxo *utxo = tal(ctx, struct utxo); - u32 *blockheight, *spendheight, *reserved_til; + u32 *blockheight, *spendheight; db_column_txid(stmt, 0, &utxo->txid); utxo->outnum = db_column_int(stmt, 1); db_column_amount_sat(stmt, 2, &utxo->amount); @@ -192,7 +191,6 @@ static struct utxo *wallet_stmt2output(const tal_t *ctx, struct db_stmt *stmt) utxo->blockheight = NULL; utxo->spendheight = NULL; - utxo->reserved_til = NULL; if (!db_column_is_null(stmt, 10)) { blockheight = tal(utxo, u32); @@ -206,11 +204,9 @@ static struct utxo *wallet_stmt2output(const tal_t *ctx, struct db_stmt *stmt) utxo->spendheight = spendheight; } - if (!db_column_is_null(stmt, 13)) { - reserved_til = tal(utxo, u32); - *reserved_til = db_column_int(stmt, 13); - utxo->reserved_til = reserved_til; - } + /* This column can be null if 0.9.1 db or below. */ + utxo->reserved_til = tal(utxo, u32); + *utxo->reserved_til = db_column_int_or_default(stmt, 13, 0); return utxo; } @@ -453,8 +449,6 @@ bool wallet_reserve_utxo(struct wallet *w, struct utxo *utxo, u32 current_height if (utxo->status == output_state_reserved) assert(utxo->reserved_til); - else - assert(!utxo->reserved_til); switch (utxo->status) { case output_state_spent: @@ -489,8 +483,7 @@ void wallet_unreserve_utxo(struct wallet *w, struct utxo *utxo, u32 current_heig if (!utxo->reserved_til) utxo->reserved_til = tal_dup(utxo, u32, ¤t_height); assert(utxo->reserved_til); - } else - assert(!utxo->reserved_til); + } if (utxo->status != output_state_reserved) fatal("UTXO %s:%u is not reserved", @@ -645,175 +638,6 @@ bool wallet_add_onchaind_utxo(struct wallet *w, return true; } -static const struct utxo **wallet_select(const tal_t *ctx, struct wallet *w, - struct amount_sat sat, - const u32 feerate_per_kw, - size_t outscriptlen, - bool may_have_change, - u32 maxheight, - struct amount_sat *satoshi_in, - struct amount_sat *fee_estimate) -{ - size_t i = 0; - struct utxo **available; - u64 weight; - size_t num_outputs = may_have_change ? 2 : 1; - const struct utxo **utxos = tal_arr(ctx, const struct utxo *, 0); - tal_add_destructor2(utxos, destroy_utxos, w); - - /* We assume < 253 inputs, and margin is tiny if we're wrong */ - weight = bitcoin_tx_core_weight(1, num_outputs) - + bitcoin_tx_output_weight(outscriptlen); - - /* Change output will be P2WPKH */ - if (may_have_change) - weight += bitcoin_tx_output_weight(BITCOIN_SCRIPTPUBKEY_P2WPKH_LEN); - - *fee_estimate = AMOUNT_SAT(0); - *satoshi_in = AMOUNT_SAT(0); - - available = wallet_get_utxos(ctx, w, output_state_available); - - for (i = 0; i < tal_count(available); i++) { - struct amount_sat needed; - struct utxo *u = tal_steal(utxos, available[i]); - - /* If we require confirmations check that we have a - * confirmation height and that it is below the required - * maxheight (current_height - minconf) */ - if (maxheight != 0 && - (!u->blockheight || *u->blockheight > maxheight)) { - tal_free(u); - continue; - } - - tal_arr_expand(&utxos, u); - - if (!wallet_update_output_status( - w, &available[i]->txid, available[i]->outnum, - output_state_available, output_state_reserved)) - fatal("Unable to reserve output"); - - weight += bitcoin_tx_simple_input_weight(u->is_p2sh); - - if (!amount_sat_add(satoshi_in, *satoshi_in, u->amount)) - fatal("Overflow in available satoshis %zu/%zu %s + %s", - i, tal_count(available), - type_to_string(tmpctx, struct amount_sat, - satoshi_in), - type_to_string(tmpctx, struct amount_sat, - &u->amount)); - - *fee_estimate = amount_tx_fee(feerate_per_kw, weight); - if (!amount_sat_add(&needed, sat, *fee_estimate)) - fatal("Overflow in fee estimate %zu/%zu %s + %s", - i, tal_count(available), - type_to_string(tmpctx, struct amount_sat, &sat), - type_to_string(tmpctx, struct amount_sat, - fee_estimate)); - if (amount_sat_greater_eq(*satoshi_in, needed)) - break; - } - tal_free(available); - - return utxos; -} - -const struct utxo **wallet_select_coins(const tal_t *ctx, struct wallet *w, - bool with_change, - struct amount_sat sat, - const u32 feerate_per_kw, - size_t outscriptlen, - u32 maxheight, - struct amount_sat *fee_estimate, - struct amount_sat *change) -{ - struct amount_sat satoshi_in; - const struct utxo **utxo; - - utxo = wallet_select(ctx, w, sat, feerate_per_kw, - outscriptlen, with_change, maxheight, - &satoshi_in, fee_estimate); - - /* Couldn't afford it? */ - if (!amount_sat_sub(change, satoshi_in, sat) - || !amount_sat_sub(change, *change, *fee_estimate)) - return tal_free(utxo); - - if (!with_change) - *change = AMOUNT_SAT(0); - - return utxo; -} - -const struct utxo **wallet_select_specific(const tal_t *ctx, struct wallet *w, - struct bitcoin_txid **txids, - u32 **outnums) -{ - size_t i, j; - struct utxo **available; - const struct utxo **utxos = tal_arr(ctx, const struct utxo*, 0); - tal_add_destructor2(utxos, destroy_utxos, w); - - available = wallet_get_utxos(ctx, w, output_state_available); - for (i = 0; i < tal_count(txids); i++) { - for (j = 0; j < tal_count(available); j++) { - - if (bitcoin_txid_eq(&available[j]->txid, txids[i]) - && available[j]->outnum == *outnums[i]) { - struct utxo *u = tal_steal(utxos, available[j]); - tal_arr_expand(&utxos, u); - - if (!wallet_update_output_status( - w, &available[j]->txid, available[j]->outnum, - output_state_available, output_state_reserved)) - fatal("Unable to reserve output"); - } - } - } - tal_free(available); - - return utxos; -} - -const struct utxo **wallet_select_all(const tal_t *ctx, struct wallet *w, - const u32 feerate_per_kw, - size_t outscriptlen, - u32 maxheight, - struct amount_sat *value, - struct amount_sat *fee_estimate) -{ - struct amount_sat satoshi_in; - const struct utxo **utxo; - - /* Huge value, but won't overflow on addition */ - utxo = wallet_select(ctx, w, AMOUNT_SAT(1ULL << 56), feerate_per_kw, - outscriptlen, false, maxheight, - &satoshi_in, fee_estimate); - - /* Can't afford fees? */ - if (!amount_sat_sub(value, satoshi_in, *fee_estimate)) - return tal_free(utxo); - - return utxo; -} - -u8 *derive_redeem_scriptsig(const tal_t *ctx, struct wallet *w, u32 keyindex) -{ - struct ext_key ext; - struct pubkey key; - - if (bip32_key_from_parent(w->bip32_base, keyindex, - BIP32_FLAG_KEY_PUBLIC, &ext) != WALLY_OK) { - fatal("Unable to derive pubkey"); - } - - if (!pubkey_from_der(ext.pub_key, PUBKEY_CMPR_LEN, &key)) - fatal("Unble to derive pubkey from DER"); - - return bitcoin_scriptsig_p2sh_p2wpkh(ctx, &key); -} - bool wallet_can_spend(struct wallet *w, const u8 *script, u32 *index, bool *output_is_p2sh) { @@ -3933,85 +3757,6 @@ const struct forwarding *wallet_forwarded_payments_get(struct wallet *w, return results; } -struct unreleased_tx *find_unreleased_tx(struct wallet *w, - const struct bitcoin_txid *txid) -{ - struct unreleased_tx *utx; - - list_for_each(&w->unreleased_txs, utx, list) { - if (bitcoin_txid_eq(txid, &utx->txid)) - return utx; - } - return NULL; -} - -static void destroy_unreleased_tx(struct unreleased_tx *utx) -{ - list_del(&utx->list); -} - -void remove_unreleased_tx(struct unreleased_tx *utx) -{ - tal_del_destructor(utx, destroy_unreleased_tx); - list_del(&utx->list); -} - -void add_unreleased_tx(struct wallet *w, struct unreleased_tx *utx) -{ - list_add_tail(&w->unreleased_txs, &utx->list); - tal_add_destructor(utx, destroy_unreleased_tx); -} - -/* These will touch the db, so need to be explicitly freed. */ -void free_unreleased_txs(struct wallet *w) -{ - struct unreleased_tx *utx; - - while ((utx = list_top(&w->unreleased_txs, struct unreleased_tx, list))) - tal_free(utx); -} - -static void process_utxo_result(struct bitcoind *bitcoind, - const struct bitcoin_tx_output *txout, - void *_utxos) -{ - struct utxo **utxos = _utxos; - enum output_status newstate = - txout == NULL ? output_state_spent : output_state_available; - - /* Don't unreserve ones which are on timers */ - if (!utxos[0]->reserved_til || newstate == output_state_spent) { - log_unusual(bitcoind->ld->wallet->log, - "wallet: reserved output %s/%u reset to %s", - type_to_string(tmpctx, struct bitcoin_txid, &utxos[0]->txid), - utxos[0]->outnum, - newstate == output_state_spent ? "spent" : "available"); - wallet_update_output_status(bitcoind->ld->wallet, - &utxos[0]->txid, utxos[0]->outnum, - utxos[0]->status, newstate); - } - - /* If we have more, resolve them too. */ - tal_arr_remove(&utxos, 0); - if (tal_count(utxos) != 0) { - bitcoind_getutxout(bitcoind, &utxos[0]->txid, utxos[0]->outnum, - process_utxo_result, utxos); - } else - tal_free(utxos); -} - -void wallet_clean_utxos(struct wallet *w, struct bitcoind *bitcoind) -{ - struct utxo **utxos = wallet_get_utxos(NULL, w, output_state_reserved); - - if (tal_count(utxos) != 0) { - bitcoind_getutxout(bitcoind, &utxos[0]->txid, utxos[0]->outnum, - process_utxo_result, - notleak_with_children(utxos)); - } else - tal_free(utxos); -} - struct wallet_transaction *wallet_transactions_get(struct wallet *w, const tal_t *ctx) { struct db_stmt *stmt; diff --git a/wallet/wallet.h b/wallet/wallet.h index 9c9566f8c7c0..73b2347e2cb2 100644 --- a/wallet/wallet.h +++ b/wallet/wallet.h @@ -48,26 +48,10 @@ struct wallet { * the blockchain. This is currently all P2WSH outputs */ struct outpointfilter *utxoset_outpoints; - /* Unreleased txs, waiting for txdiscard/txsend */ - struct list_head unreleased_txs; - /* How many keys should we look ahead at most? */ u64 keyscan_gap; }; -/* A transaction we've txprepared, but haven't signed and released yet */ -struct unreleased_tx { - /* In wallet->unreleased_txs */ - struct list_node list; - /* All the utxos. */ - struct wallet_tx *wtx; - /* Outputs(scriptpubkey and satoshi) this pays to. */ - struct bitcoin_tx_output **outputs; - /* The tx itself (unsigned initially) */ - struct bitcoin_tx *tx; - struct bitcoin_txid txid; -}; - /* Possible states for tracked outputs in the database. Not sure yet * whether we really want to have reservations reflected in the * database, it would simplify queries at the cost of some IO ops */ @@ -440,30 +424,6 @@ struct utxo *wallet_utxo_get(const tal_t *ctx, struct wallet *w, const struct bitcoin_txid *txid, u32 outnum); -const struct utxo **wallet_select_coins(const tal_t *ctx, struct wallet *w, - bool with_change, - struct amount_sat value, - const u32 feerate_per_kw, - size_t outscriptlen, - u32 maxheight, - struct amount_sat *fee_estimate, - struct amount_sat *change_satoshi); - -const struct utxo **wallet_select_all(const tal_t *ctx, struct wallet *w, - const u32 feerate_per_kw, - size_t outscriptlen, - u32 maxheight, - struct amount_sat *sat, - struct amount_sat *fee_estimate); - -/* derive_redeem_scriptsig - Compute the scriptSig for a P2SH-P2WPKH - * - * @ctx - allocation context - * @w - wallet - * @keyindex - index of the internal BIP32 key - */ -u8 *derive_redeem_scriptsig(const tal_t *ctx, struct wallet *w, u32 keyindex); - /** * wallet_select_specific - Select utxos given an array of txids and an array of outputs index * diff --git a/wallet/walletrpc.c b/wallet/walletrpc.c index 2b2db663c3d1..c1c69c59b354 100644 --- a/wallet/walletrpc.c +++ b/wallet/walletrpc.c @@ -15,8 +15,6 @@ #include #include #include -#include -#include #include #include #include @@ -35,283 +33,6 @@ #include #include -struct tx_broadcast { - struct command *cmd; - const struct utxo **utxos; - const struct wally_tx *wtx; - struct amount_sat *expected_change; -}; - -static struct tx_broadcast *unreleased_tx_to_broadcast(const tal_t *ctx, - struct command *cmd, - struct unreleased_tx *utx) -{ - struct tx_broadcast *txb = tal(ctx, struct tx_broadcast); - struct amount_sat *change = tal(txb, struct amount_sat); - - txb->cmd = cmd; - txb->utxos = utx->wtx->utxos; - txb->wtx = utx->tx->wtx; - *change = utx->wtx->change; - txb->expected_change = change; - return txb; -} - -/** - * wallet_withdrawal_broadcast - The tx has been broadcast (or it failed) - * - * This is the final step in the withdrawal. We either successfully - * broadcast the withdrawal transaction or it failed somehow. So we - * report success or a broadcast failure. Upon success we also mark - * the used outputs as spent, and add the change output to our pool of - * available outputs. - */ -static void wallet_withdrawal_broadcast(struct bitcoind *bitcoind UNUSED, - bool success, const char *msg, - struct tx_broadcast *txb) -{ - struct command *cmd = txb->cmd; - struct lightningd *ld = cmd->ld; - - /* FIXME: This won't be necessary once we use ccan/json_out! */ - /* Massage output into shape so it doesn't kill the JSON serialization */ - char *output = tal_strjoin(cmd, tal_strsplit(cmd, msg, "\n", STR_NO_EMPTY), " ", STR_NO_TRAIL); - if (success) { - struct bitcoin_txid txid; - struct amount_sat change = AMOUNT_SAT(0); - - /* Mark used outputs as spent */ - wallet_confirm_utxos(ld->wallet, txb->utxos); - - /* Extract the change output and add it to the DB */ - wallet_extract_owned_outputs(ld->wallet, txb->wtx, NULL, &change); - - /* Note normally, change_satoshi == withdraw->wtx->change, but - * not if we're actually making a payment to ourselves! */ - if (txb->expected_change) - assert(amount_sat_greater_eq(change, *txb->expected_change)); - - struct json_stream *response = json_stream_success(cmd); - wally_txid(txb->wtx, &txid); - json_add_hex_talarr(response, "tx", - linearize_wtx(tmpctx, txb->wtx)); - json_add_txid(response, "txid", &txid); - was_pending(command_success(cmd, response)); - } else { - was_pending(command_fail(cmd, LIGHTNINGD, - "Error broadcasting transaction: %s. Unsent tx discarded %s", - output, - type_to_string(tmpctx, struct wally_tx, txb->wtx))); - } -} - -/* Signs the tx, broadcasts it: broadcast calls wallet_withdrawal_broadcast */ -static struct command_result *broadcast_and_wait(struct command *cmd, - struct unreleased_tx *utx) -{ - struct wally_psbt *signed_psbt; - struct wally_tx *signed_wtx; - struct bitcoin_txid signed_txid; - - /* FIXME: hsm will sign almost anything, but it should really - * fail cleanly (not abort!) and let us report the error here. */ - u8 *msg = towire_hsmd_sign_withdrawal(cmd, utx->wtx->utxos, utx->tx->psbt); - - if (!wire_sync_write(cmd->ld->hsm_fd, take(msg))) - fatal("Could not write sign_withdrawal to HSM: %s", - strerror(errno)); - - msg = wire_sync_read(cmd, cmd->ld->hsm_fd); - - if (!fromwire_hsmd_sign_withdrawal_reply(utx, msg, &signed_psbt)) - fatal("HSM gave bad sign_withdrawal_reply %s", - tal_hex(tmpctx, msg)); - - signed_wtx = psbt_finalize(signed_psbt, true); - - if (!signed_wtx) { - /* Have the utx persist past this command */ - tal_steal(cmd->ld->wallet, utx); - add_unreleased_tx(cmd->ld->wallet, utx); - return command_fail(cmd, LIGHTNINGD, - "PSBT is not finalized %s", - type_to_string(tmpctx, - struct wally_psbt, - signed_psbt)); - } - - /* Sanity check */ - wally_txid(signed_wtx, &signed_txid); - if (!bitcoin_txid_eq(&signed_txid, &utx->txid)) - fatal("HSM changed txid: unsigned %s, signed %s", - tal_hex(tmpctx, linearize_tx(tmpctx, utx->tx)), - tal_hex(tmpctx, linearize_wtx(tmpctx, signed_wtx))); - - /* Replace unsigned tx by signed tx. */ - wally_tx_free(utx->tx->wtx); - utx->tx->wtx = tal_steal(utx->tx, signed_wtx); - tal_free(utx->tx->psbt); - utx->tx->psbt = tal_steal(utx->tx, signed_psbt); - - /* Now broadcast the transaction */ - bitcoind_sendrawtx(cmd->ld->topology->bitcoind, - tal_hex(tmpctx, linearize_tx(tmpctx, utx->tx)), - wallet_withdrawal_broadcast, - unreleased_tx_to_broadcast(cmd, cmd, utx)); - - return command_still_pending(cmd); -} - -/* Parsing code for withdraw. - * - * Returns NULL on success, and fills in wtx, output and - * maybe changekey (owned by cmd). Otherwise, cmd has failed, so don't - * access it! (It's been freed). */ -static struct command_result *json_prepare_tx(struct command *cmd, - const char *buffer, - const jsmntok_t *params, - struct unreleased_tx **utx) -{ - u32 *feerate_per_kw = NULL; - struct command_result *result; - u32 *minconf, maxheight; - struct pubkey *changekey; - struct bitcoin_tx_output **outputs; - const u8 *destination = NULL; - size_t out_len; - const struct utxo **chosen_utxos = NULL; - u32 locktime; - - *utx = tal(cmd, struct unreleased_tx); - (*utx)->wtx = tal(*utx, struct wallet_tx); - wtx_init(cmd, (*utx)->wtx, AMOUNT_SAT(-1ULL)); - - /* *withdraw* command still use 'destination' and 'satoshi' as parameters. */ - if (!param(cmd, buffer, params, - p_req("destination", param_bitcoin_address, - &destination), - p_req("satoshi", param_wtx, (*utx)->wtx), - p_opt("feerate", param_feerate, &feerate_per_kw), - p_opt_def("minconf", param_number, &minconf, 1), - p_opt("utxos", param_utxos, &chosen_utxos), - NULL)) - return command_param_failed(); - - /* Setting the locktime to the next block to be mined has multiple - * benefits: - * - anti fee-snipping (even if not yet likely) - * - less distinguishable transactions (with this we create - * general-purpose transactions which looks like bitcoind: - * native segwit, nlocktime set to tip, and sequence set to - * 0xFFFFFFFE by default. Other wallets are likely to implement - * this too). - */ - locktime = cmd->ld->topology->tip->height; - /* Eventually fuzz it too. */ - if (pseudorand(10) == 0) - locktime -= (u32)pseudorand(100); - - if (!feerate_per_kw) { - /* We mainly use `txprepare` for opening transactions, and FEERATE_OPENING - * is kind of the new FEERATE_NORMAL so it fits well `withdraw` too. */ - result = param_feerate_estimate(cmd, &feerate_per_kw, - FEERATE_OPENING); - if (result) - return result; - } - - maxheight = minconf_to_maxheight(*minconf, cmd->ld); - - outputs = tal_arr(tmpctx, struct bitcoin_tx_output *, 1); - outputs[0] = new_tx_output(outputs, (*utx)->wtx->amount, - destination); - out_len = tal_count(outputs[0]->script); - - if (chosen_utxos) - result = wtx_from_utxos((*utx)->wtx, *feerate_per_kw, - out_len, maxheight, - chosen_utxos); - else - result = wtx_select_utxos((*utx)->wtx, *feerate_per_kw, - out_len, maxheight); - - if (result) - return result; - - /* Because of the max limit of AMOUNT_SAT(-1ULL), - * `(*utx)->wtx->all_funds` won't change in `wtx_select_utxos()` */ - if ((*utx)->wtx->all_funds) - outputs[0]->amount = (*utx)->wtx->amount; - - /* Add the change as the last output */ - if (!amount_sat_eq((*utx)->wtx->change, AMOUNT_SAT(0))) { - struct bitcoin_tx_output *change_output; - - changekey = tal(tmpctx, struct pubkey); - if (!bip32_pubkey(cmd->ld->wallet->bip32_base, changekey, - (*utx)->wtx->change_key_index)) - return command_fail(cmd, LIGHTNINGD, "Keys generation failure"); - - change_output = new_tx_output(outputs, (*utx)->wtx->change, - scriptpubkey_p2wpkh(tmpctx, changekey)); - tal_arr_expand(&outputs, change_output); - } - - (*utx)->outputs = tal_steal(*utx, outputs); - (*utx)->tx = withdraw_tx(*utx, chainparams, - (*utx)->wtx->utxos, - (*utx)->outputs, - cmd->ld->wallet->bip32_base, - /* FIXME: Should probably be - * struct abs_locktime. - */ - locktime); - - bitcoin_txid((*utx)->tx, &(*utx)->txid); - return NULL; -} - -/** - * json_withdraw - Entrypoint for the withdrawal flow - * - * A user has requested a withdrawal over the JSON-RPC, parse the - * request, select coins and a change key. Then send the request to - * the HSM to generate the signatures. - */ -static struct command_result *json_withdraw(struct command *cmd, - const char *buffer, - const jsmntok_t *obj UNNEEDED, - const jsmntok_t *params) -{ - struct unreleased_tx *utx; - struct command_result *res; - - res = json_prepare_tx(cmd, buffer, params, &utx); - if (res) - return res; - - /* Store the transaction in the DB and annotate it as a withdrawal */ - wallet_transaction_add(cmd->ld->wallet, utx->tx->wtx, 0, 0); - wallet_transaction_annotate(cmd->ld->wallet, &utx->txid, - TX_WALLET_WITHDRAWAL, 0); - - return broadcast_and_wait(cmd, utx); -} - -static const struct json_command withdraw_command = { - "withdraw", - "bitcoin", - json_withdraw, - "Send to {destination} address {satoshi} (or 'all') amount via Bitcoin " - "transaction, at optional {feerate}", - false, - "Send funds from the internal wallet to the specified address. Either " - "specify a number of satoshis to send or 'all' to sweep all funds in the " - "internal wallet to the address. Only use outputs that have at least " - "{minconf} confirmations." -}; -AUTODATA(json_command, &withdraw_command); - /* May return NULL if encoding error occurs. */ static char * encode_pubkey_to_addr(const tal_t *ctx, From ea819107eba91983c6b86b666b493c95494a29ae Mon Sep 17 00:00:00 2001 From: Rusty Russell Date: Fri, 28 Aug 2020 13:26:34 +0930 Subject: [PATCH 22/30] common: remove funding_tx. It's now only needed by devtools/mkfunding, so include a reduced one there, and this also means we remove tx_spending_utxos(). Signed-off-by: Rusty Russell --- common/Makefile | 1 - common/funding_tx.c | 56 --------- common/funding_tx.h | 47 ------- common/test/run-funding_tx.c | 229 ----------------------------------- common/utxo.c | 57 --------- common/utxo.h | 10 -- devtools/Makefile | 4 +- devtools/mkfunding.c | 50 ++++++-- hsmd/Makefile | 1 - hsmd/hsmd.c | 1 - lightningd/Makefile | 1 - lightningd/opening_control.c | 1 - openingd/Makefile | 1 - openingd/openingd.c | 1 - 14 files changed, 42 insertions(+), 418 deletions(-) delete mode 100644 common/funding_tx.c delete mode 100644 common/funding_tx.h delete mode 100644 common/test/run-funding_tx.c diff --git a/common/Makefile b/common/Makefile index b62e738c08f0..c7139afc1d44 100644 --- a/common/Makefile +++ b/common/Makefile @@ -26,7 +26,6 @@ COMMON_SRC_NOGEN := \ common/ecdh_hsmd.c \ common/features.c \ common/fee_states.c \ - common/funding_tx.c \ common/gossip_rcvd_filter.c \ common/gossip_store.c \ common/hash_u5.c \ diff --git a/common/funding_tx.c b/common/funding_tx.c deleted file mode 100644 index cc6111029a60..000000000000 --- a/common/funding_tx.c +++ /dev/null @@ -1,56 +0,0 @@ -#include "funding_tx.h" -#include -#include -#include -#include -#include -#include -#include - -#ifndef SUPERVERBOSE -#define SUPERVERBOSE(...) -#endif - -struct bitcoin_tx *funding_tx(const tal_t *ctx, - const struct chainparams *chainparams, - u16 *outnum, - const struct utxo **utxomap, - struct amount_sat funding, - const struct pubkey *local_fundingkey, - const struct pubkey *remote_fundingkey, - struct amount_sat change, - const struct pubkey *changekey, - const struct ext_key *bip32_base) -{ - u8 *wscript; - struct bitcoin_tx *tx; - bool has_change = !amount_sat_eq(change, AMOUNT_SAT(0)); - - tx = tx_spending_utxos(ctx, chainparams, utxomap, bip32_base, - has_change, 1, 0, BITCOIN_TX_DEFAULT_SEQUENCE); - - - wscript = bitcoin_redeem_2of2(tx, local_fundingkey, remote_fundingkey); - SUPERVERBOSE("# funding witness script = %s\n", - tal_hex(wscript, wscript)); - bitcoin_tx_add_output(tx, scriptpubkey_p2wsh(tx, wscript), wscript, funding); - tal_free(wscript); - - if (has_change) { - const void *map[2]; - map[0] = int2ptr(0); - map[1] = int2ptr(1); - bitcoin_tx_add_output(tx, scriptpubkey_p2wpkh(tx, changekey), - NULL, change); - permute_outputs(tx, NULL, map); - *outnum = (map[0] == int2ptr(0) ? 0 : 1); - } else { - *outnum = 0; - } - - permute_inputs(tx, (const void **)utxomap); - - bitcoin_tx_finalize(tx); - assert(bitcoin_tx_check(tx)); - return tx; -} diff --git a/common/funding_tx.h b/common/funding_tx.h deleted file mode 100644 index 577a4cafb2ee..000000000000 --- a/common/funding_tx.h +++ /dev/null @@ -1,47 +0,0 @@ -#ifndef LIGHTNING_COMMON_FUNDING_TX_H -#define LIGHTNING_COMMON_FUNDING_TX_H -#include "config.h" -#include -#include -#include -#include - -struct bitcoin_tx; -struct ext_key; -struct privkey; -struct pubkey; -struct utxo; - -/** - * funding_tx: create a P2WSH funding transaction for a channel. - * @ctx: context to tal from. - * @chainparams: (in) the params for the resulting transaction. - * @outnum: (out) txout (0 or 1) which is the funding output. - * @utxomap: (in/out) tal_arr of UTXO pointers to spend (permuted to match) - * @funding: (in) satoshis to output. - * @local_fundingkey: (in) local key for 2of2 funding output. - * @remote_fundingkey: (in) remote key for 2of2 funding output. - * @change: (in) amount to send as change. - * @changekey: (in) key to send change to (only used if change_satoshis != 0). - * @bip32_base: (in) bip32 base for key derivation, or NULL. - * - * If bip32_base is supplied, scriptSig will be added for p2sh inputs: this - * means our signing code will fail, but txid will be correct. If NULL, - * the txid will be incorrect, by signing will succeed. - * - * This is done because all other txs have no scriptSig (being pure Segwit) - * so our signature code simply asserts there's no scriptsig (which would - * have to be removed for signing anyway). The funding transaction is - * a special case because of the P2SH inputs. - */ -struct bitcoin_tx *funding_tx(const tal_t *ctx, - const struct chainparams *chainparams, - u16 *outnum, - const struct utxo **utxomap, - struct amount_sat funding, - const struct pubkey *local_fundingkey, - const struct pubkey *remote_fundingkey, - struct amount_sat change, - const struct pubkey *changekey, - const struct ext_key *bip32_base); -#endif /* LIGHTNING_COMMON_FUNDING_TX_H */ diff --git a/common/test/run-funding_tx.c b/common/test/run-funding_tx.c deleted file mode 100644 index f57a7149b892..000000000000 --- a/common/test/run-funding_tx.c +++ /dev/null @@ -1,229 +0,0 @@ -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include "../amount.c" -#define SUPERVERBOSE printf -#include "../funding_tx.c" -#undef SUPERVERBOSE -#include "../key_derive.c" -#include "../type_to_string.c" -#include "../permute_tx.c" -#include "../utxo.c" - -/* AUTOGENERATED MOCKS START */ -/* Generated stub for fromwire */ -const u8 *fromwire(const u8 **cursor UNNEEDED, size_t *max UNNEEDED, void *copy UNNEEDED, size_t n UNNEEDED) -{ fprintf(stderr, "fromwire called!\n"); abort(); } -/* Generated stub for fromwire_bool */ -bool fromwire_bool(const u8 **cursor UNNEEDED, size_t *max UNNEEDED) -{ fprintf(stderr, "fromwire_bool called!\n"); abort(); } -/* Generated stub for fromwire_fail */ -void *fromwire_fail(const u8 **cursor UNNEEDED, size_t *max UNNEEDED) -{ fprintf(stderr, "fromwire_fail called!\n"); abort(); } -/* Generated stub for fromwire_node_id */ -void fromwire_node_id(const u8 **cursor UNNEEDED, size_t *max UNNEEDED, struct node_id *id UNNEEDED) -{ fprintf(stderr, "fromwire_node_id called!\n"); abort(); } -/* Generated stub for fromwire_secp256k1_ecdsa_signature */ -void fromwire_secp256k1_ecdsa_signature(const u8 **cursor UNNEEDED, size_t *max UNNEEDED, - secp256k1_ecdsa_signature *signature UNNEEDED) -{ fprintf(stderr, "fromwire_secp256k1_ecdsa_signature called!\n"); abort(); } -/* Generated stub for fromwire_sha256 */ -void fromwire_sha256(const u8 **cursor UNNEEDED, size_t *max UNNEEDED, struct sha256 *sha256 UNNEEDED) -{ fprintf(stderr, "fromwire_sha256 called!\n"); abort(); } -/* Generated stub for fromwire_tal_arrn */ -u8 *fromwire_tal_arrn(const tal_t *ctx UNNEEDED, - const u8 **cursor UNNEEDED, size_t *max UNNEEDED, size_t num UNNEEDED) -{ fprintf(stderr, "fromwire_tal_arrn called!\n"); abort(); } -/* Generated stub for fromwire_u16 */ -u16 fromwire_u16(const u8 **cursor UNNEEDED, size_t *max UNNEEDED) -{ fprintf(stderr, "fromwire_u16 called!\n"); abort(); } -/* Generated stub for fromwire_u32 */ -u32 fromwire_u32(const u8 **cursor UNNEEDED, size_t *max UNNEEDED) -{ fprintf(stderr, "fromwire_u32 called!\n"); abort(); } -/* Generated stub for fromwire_u64 */ -u64 fromwire_u64(const u8 **cursor UNNEEDED, size_t *max UNNEEDED) -{ fprintf(stderr, "fromwire_u64 called!\n"); abort(); } -/* Generated stub for fromwire_u8 */ -u8 fromwire_u8(const u8 **cursor UNNEEDED, size_t *max UNNEEDED) -{ fprintf(stderr, "fromwire_u8 called!\n"); abort(); } -/* Generated stub for towire */ -void towire(u8 **pptr UNNEEDED, const void *data UNNEEDED, size_t len UNNEEDED) -{ fprintf(stderr, "towire called!\n"); abort(); } -/* Generated stub for towire_bool */ -void towire_bool(u8 **pptr UNNEEDED, bool v UNNEEDED) -{ fprintf(stderr, "towire_bool called!\n"); abort(); } -/* Generated stub for towire_node_id */ -void towire_node_id(u8 **pptr UNNEEDED, const struct node_id *id UNNEEDED) -{ fprintf(stderr, "towire_node_id called!\n"); abort(); } -/* Generated stub for towire_secp256k1_ecdsa_signature */ -void towire_secp256k1_ecdsa_signature(u8 **pptr UNNEEDED, - const secp256k1_ecdsa_signature *signature UNNEEDED) -{ fprintf(stderr, "towire_secp256k1_ecdsa_signature called!\n"); abort(); } -/* Generated stub for towire_sha256 */ -void towire_sha256(u8 **pptr UNNEEDED, const struct sha256 *sha256 UNNEEDED) -{ fprintf(stderr, "towire_sha256 called!\n"); abort(); } -/* Generated stub for towire_u16 */ -void towire_u16(u8 **pptr UNNEEDED, u16 v UNNEEDED) -{ fprintf(stderr, "towire_u16 called!\n"); abort(); } -/* Generated stub for towire_u32 */ -void towire_u32(u8 **pptr UNNEEDED, u32 v UNNEEDED) -{ fprintf(stderr, "towire_u32 called!\n"); abort(); } -/* Generated stub for towire_u64 */ -void towire_u64(u8 **pptr UNNEEDED, u64 v UNNEEDED) -{ fprintf(stderr, "towire_u64 called!\n"); abort(); } -/* Generated stub for towire_u8 */ -void towire_u8(u8 **pptr UNNEEDED, u8 v UNNEEDED) -{ fprintf(stderr, "towire_u8 called!\n"); abort(); } -/* Generated stub for towire_u8_array */ -void towire_u8_array(u8 **pptr UNNEEDED, const u8 *arr UNNEEDED, size_t num UNNEEDED) -{ fprintf(stderr, "towire_u8_array called!\n"); abort(); } -/* AUTOGENERATED MOCKS END */ - -#if 0 -static struct sha256 sha256_from_hex(const char *hex) -{ - struct sha256 sha256; - if (strstarts(hex, "0x")) - hex += 2; - if (!hex_decode(hex, strlen(hex), &sha256, sizeof(sha256))) - abort(); - return sha256; -} - -static struct privkey privkey_from_hex(const char *hex) -{ - struct privkey pk; - size_t len; - if (strstarts(hex, "0x")) - hex += 2; - len = strlen(hex); - if (len == 66 && strends(hex, "01")) - len -= 2; - if (!hex_decode(hex, len, &pk, sizeof(pk))) - abort(); - return pk; -} -#endif - -int main(int argc, const char *argv[]) -{ - common_setup(argv[0]); - - struct bitcoin_tx *input, *funding; - struct amount_sat fee, change; - struct pubkey local_funding_pubkey, remote_funding_pubkey; - struct privkey input_privkey; - struct pubkey inputkey; - bool testnet; - struct utxo utxo; - const struct utxo **utxomap; - struct amount_sat funding_sat; - u16 funding_outnum; - u8 *subscript, *script; - struct bitcoin_signature sig; - struct bitcoin_address addr; - struct amount_sat tmpamt; - struct amount_asset asset; - - chainparams = chainparams_for_network("bitcoin"); - - /* BOLT #3: - * - * Block 1 coinbase transaction: 01000000010000000000000000000000000000000000000000000000000000000000000000ffffffff03510101ffffffff0100f2052a010000001976a9143ca33c2e4446f4a305f23c80df8ad1afdcf652f988ac00000000 - */ - input = bitcoin_tx_from_hex(tmpctx, - "01000000010000000000000000000000000000000000000000000000000000000000000000ffffffff03510101ffffffff0100f2052a010000001976a9143ca33c2e4446f4a305f23c80df8ad1afdcf652f988ac00000000", - strlen("01000000010000000000000000000000000000000000000000000000000000000000000000ffffffff03510101ffffffff0100f2052a010000001976a9143ca33c2e4446f4a305f23c80df8ad1afdcf652f988ac00000000")); - input->chainparams = chainparams_for_network("bitcoin"); - assert(input); - - /* BOLT #3: - * Block 1 coinbase privkey: 6bd078650fcee8444e4e09825227b801a1ca928debb750eb36e6d56124bb20e801 - * # privkey in base58: cRCH7YNcarfvaiY1GWUKQrRGmoezvfAiqHtdRvxe16shzbd7LDMz - */ - if (!key_from_base58("cRCH7YNcarfvaiY1GWUKQrRGmoezvfAiqHtdRvxe16shzbd7LDMz", strlen("cRCH7YNcarfvaiY1GWUKQrRGmoezvfAiqHtdRvxe16shzbd7LDMz"), - &testnet, &input_privkey, &inputkey)) - abort(); - assert(testnet); - printf("* Block 1 coinbase privkey: %s\n", - type_to_string(tmpctx, struct privkey, &input_privkey)); - - /* BOLT #3: - * - * The funding transaction is paid to the following pubkeys: - * - * local_funding_pubkey: 023da092f6980e58d2c037173180e9a465476026ee50f96695963e8efe436f54eb - * remote_funding_pubkey: 030e9f7b623d2ccc7c9bd44d66d5ce21ce504c0acf6385a132cec6d3c39fa711c1 - */ - if (!pubkey_from_hexstr("023da092f6980e58d2c037173180e9a465476026ee50f96695963e8efe436f54eb", - strlen("023da092f6980e58d2c037173180e9a465476026ee50f96695963e8efe436f54eb"), - &local_funding_pubkey)) - abort(); - if (!pubkey_from_hexstr("030e9f7b623d2ccc7c9bd44d66d5ce21ce504c0acf6385a132cec6d3c39fa711c1", - strlen("030e9f7b623d2ccc7c9bd44d66d5ce21ce504c0acf6385a132cec6d3c39fa711c1"), - &remote_funding_pubkey)) - abort(); - - bitcoin_txid(input, &utxo.txid); - utxo.outnum = 0; - utxo.amount = AMOUNT_SAT(5000000000); - utxo.is_p2sh = false; - utxo.close_info = NULL; - utxo.scriptPubkey = tal_hexdata(tmpctx, "a914a5996075e4b468c9fb01d131bf9d4052a6fee19e87", strlen("a914a5996075e4b468c9fb01d131bf9d4052a6fee19e87")); - funding_sat = AMOUNT_SAT(10000000); - fee = AMOUNT_SAT(13920); - - printf("input[0] txid: %s\n", - tal_hexstr(tmpctx, &utxo.txid, sizeof(utxo.txid))); - printf("input[0] input: %u\n", utxo.outnum); - printf("input[0] satoshis: %s\n", - type_to_string(tmpctx, struct amount_sat, &utxo.amount)); - printf("funding: %s\n", - type_to_string(tmpctx, struct amount_sat, &funding_sat)); - - utxomap = tal_arr(tmpctx, const struct utxo *, 1); - utxomap[0] = &utxo; - if (!amount_sat_sub(&change, utxo.amount, funding_sat) - || !amount_sat_sub(&change, change, fee)) - abort(); - funding = funding_tx(tmpctx, chainparams, - &funding_outnum, utxomap, - funding_sat, - &local_funding_pubkey, - &remote_funding_pubkey, - change, - &inputkey, NULL); - printf("# fee: %s\n", - type_to_string(tmpctx, struct amount_sat, &fee)); - - asset = bitcoin_tx_output_get_amount(funding, !funding_outnum); - assert(amount_asset_is_main(&asset)); - tmpamt = amount_asset_to_sat(&asset); - printf("change: %s\n", - type_to_string(tmpctx, struct amount_sat, - &tmpamt)); - - printf("funding output: %u\n", funding_outnum); - - pubkey_to_hash160(&inputkey, &addr.addr); - subscript = scriptpubkey_p2pkh(funding, &addr); - sign_tx_input(funding, 0, subscript, NULL, &input_privkey, &inputkey, - SIGHASH_ALL, &sig); - - script = bitcoin_redeem_p2pkh(funding, &inputkey, &sig); - bitcoin_tx_input_set_script(funding, 0, script); - printf("funding tx: %s\n", - tal_hex(tmpctx, linearize_tx(tmpctx, funding))); - - /* No memory leaks please */ - common_shutdown(); - return 0; -} diff --git a/common/utxo.c b/common/utxo.c index 5f78cc69e620..25fa4dd5963c 100644 --- a/common/utxo.c +++ b/common/utxo.c @@ -64,63 +64,6 @@ struct utxo *fromwire_utxo(const tal_t *ctx, const u8 **ptr, size_t *max) return utxo; } -struct bitcoin_tx *tx_spending_utxos(const tal_t *ctx, - const struct chainparams *chainparams, - const struct utxo **utxos, - const struct ext_key *bip32_base, - bool add_change_output, - size_t num_output, - u32 nlocktime, - u32 nsequence) -{ - struct pubkey key; - u8 *scriptSig, *redeemscript; - - size_t outcount = add_change_output ? 1 + num_output : num_output; - struct bitcoin_tx *tx = bitcoin_tx(ctx, chainparams, tal_count(utxos), - outcount, nlocktime); - - for (size_t i = 0; i < tal_count(utxos); i++) { - u32 this_nsequence; - if (utxos[i]->is_p2sh && bip32_base) { - bip32_pubkey(bip32_base, &key, utxos[i]->keyindex); - scriptSig = - bitcoin_scriptsig_p2sh_p2wpkh(tmpctx, &key); - redeemscript = - bitcoin_redeem_p2sh_p2wpkh(tmpctx, &key); - - } else { - scriptSig = NULL; - redeemscript = NULL; - } - - /* BOLT-a12da24dd0102c170365124782b46d9710950ac1 #3: - * #### `to_remote` Output - * ... - * The output is spent by a transaction with `nSequence` field - * set to `1` and witness: - */ - if (utxos[i]->close_info && utxos[i]->close_info->option_anchor_outputs) - this_nsequence = 1; - else - this_nsequence = nsequence; - - bitcoin_tx_add_input(tx, &utxos[i]->txid, - utxos[i]->outnum, - this_nsequence, - scriptSig, utxos[i]->amount, - utxos[i]->scriptPubkey, NULL); - - /* Add redeemscript to the PSBT input */ - if (redeemscript) - psbt_input_set_redeemscript(tx->psbt, i, - redeemscript); - - } - - return tx; -} - size_t utxo_spend_weight(const struct utxo *utxo) { return bitcoin_tx_simple_input_weight(utxo->is_p2sh); diff --git a/common/utxo.h b/common/utxo.h index 527ebee0304f..4dd1c1642aed 100644 --- a/common/utxo.h +++ b/common/utxo.h @@ -50,16 +50,6 @@ struct utxo { void towire_utxo(u8 **pptr, const struct utxo *utxo); struct utxo *fromwire_utxo(const tal_t *ctx, const u8 **ptr, size_t *max); -/* Create a tx, and populate inputs from utxos */ -struct bitcoin_tx *tx_spending_utxos(const tal_t *ctx, - const struct chainparams *chainparams, - const struct utxo **utxos, - const struct ext_key *bip32_base, - bool add_change_output, - size_t num_output, - u32 nlocktime, - u32 nsequence); - /* Estimate of (signed) UTXO weight in transaction */ size_t utxo_spend_weight(const struct utxo *utxo); #endif /* LIGHTNING_COMMON_UTXO_H */ diff --git a/devtools/Makefile b/devtools/Makefile index d55ee1616113..0034f393ff62 100644 --- a/devtools/Makefile +++ b/devtools/Makefile @@ -67,11 +67,11 @@ $(DEVTOOLS_OBJS) $(DEVTOOLS_TOOL_OBJS): wire/wire.h devtools/mkcommit: $(DEVTOOLS_COMMON_OBJS) $(CCAN_OBJS) $(BITCOIN_OBJS) common/derive_basepoints.o common/keyset.o common/key_derive.o common/initial_commit_tx.o common/permute_tx.o wire/fromwire.o wire/towire.o devtools/mkcommit.o channeld/full_channel.o common/initial_channel.o common/htlc_state.o common/pseudorand.o common/htlc_tx.o channeld/commit_tx.o common/htlc_trim.o -devtools/mkfunding: $(DEVTOOLS_COMMON_OBJS) $(CCAN_OBJS) $(BITCOIN_OBJS) wire/fromwire.o wire/towire.o common/funding_tx.o common/utxo.o common/permute_tx.o common/key_derive.o devtools/mkfunding.o +devtools/mkfunding: $(DEVTOOLS_COMMON_OBJS) $(CCAN_OBJS) $(BITCOIN_OBJS) wire/fromwire.o wire/towire.o common/key_derive.o devtools/mkfunding.o devtools/mkclose: $(DEVTOOLS_COMMON_OBJS) $(CCAN_OBJS) $(BITCOIN_OBJS) wire/fromwire.o wire/towire.o devtools/mkclose.o -devtools/mkgossip: $(DEVTOOLS_COMMON_OBJS) $(CCAN_OBJS) $(BITCOIN_OBJS) wire/fromwire.o wire/towire.o common/funding_tx.o common/utxo.o common/permute_tx.o common/key_derive.o devtools/mkgossip.o +devtools/mkgossip: $(DEVTOOLS_COMMON_OBJS) $(CCAN_OBJS) $(BITCOIN_OBJS) wire/fromwire.o wire/towire.o common/utxo.o common/permute_tx.o common/key_derive.o devtools/mkgossip.o devtools/mkencoded: $(DEVTOOLS_COMMON_OBJS) $(CCAN_OBJS) $(BITCOIN_OBJS) wire/fromwire.o wire/towire.o devtools/mkencoded.o diff --git a/devtools/mkfunding.c b/devtools/mkfunding.c index dcc37a425987..507d7874b501 100644 --- a/devtools/mkfunding.c +++ b/devtools/mkfunding.c @@ -13,7 +13,6 @@ #include #include #include -#include #include #include #include @@ -37,6 +36,44 @@ static char *sig_as_hex(const struct bitcoin_signature *sig) return tal_hexstr(NULL, compact_sig, sizeof(compact_sig)); } +static struct bitcoin_tx *tx_spending_utxo(const tal_t *ctx, + const struct utxo *utxo, + size_t num_output, + u32 nlocktime, + u32 nsequence) +{ + struct bitcoin_tx *tx = bitcoin_tx(ctx, chainparams, 1, num_output, + nlocktime); + + assert(!utxo->is_p2sh); + bitcoin_tx_add_input(tx, &utxo->txid, utxo->outnum, + nsequence, NULL, utxo->amount, + utxo->scriptPubkey, NULL); + + return tx; +} + +static struct bitcoin_tx *funding_tx(const tal_t *ctx, + const struct utxo *utxo, + struct amount_sat funding, + const struct pubkey *local_fundingkey, + const struct pubkey *remote_fundingkey) +{ + u8 *wscript; + struct bitcoin_tx *tx; + + tx = tx_spending_utxo(ctx, utxo, + 1, 0, BITCOIN_TX_DEFAULT_SEQUENCE); + + wscript = bitcoin_redeem_2of2(tx, local_fundingkey, remote_fundingkey); + bitcoin_tx_add_output(tx, scriptpubkey_p2wsh(tx, wscript), wscript, funding); + tal_free(wscript); + + bitcoin_tx_finalize(tx); + assert(bitcoin_tx_check(tx)); + return tx; +} + int main(int argc, char *argv[]) { struct privkey input_privkey; @@ -46,10 +83,8 @@ int main(int argc, char *argv[]) unsigned int feerate_per_kw; int argnum; struct bitcoin_tx *tx; - u16 outnum; size_t weight; struct utxo input; - const struct utxo **utxomap; struct bitcoin_signature sig; struct bitcoin_txid txid; u8 **witnesses; @@ -111,14 +146,9 @@ int main(int argc, char *argv[]) type_to_string(NULL, struct amount_sat, &input.amount), type_to_string(NULL, struct amount_sat, &fee)); - /* funding_tx wants a utxo array. */ - utxomap = tal_arr(tmpctx, const struct utxo *, 1); - utxomap[0] = &input; - /* No change output, so we don't need a bip32 base. */ - tx = funding_tx(NULL, chainparams, &outnum, utxomap, funding_amount, - &funding_localkey, &funding_remotekey, - AMOUNT_SAT(0), NULL, NULL); + tx = funding_tx(NULL, &input, funding_amount, + &funding_localkey, &funding_remotekey); /* P2WSH of inputkey */ bitcoin_tx_input_set_script(tx, 0, NULL); diff --git a/hsmd/Makefile b/hsmd/Makefile index f3ad5ca72fd4..e72ab99d5fb5 100644 --- a/hsmd/Makefile +++ b/hsmd/Makefile @@ -24,7 +24,6 @@ HSMD_COMMON_OBJS := \ common/daemon.o \ common/daemon_conn.o \ common/derive_basepoints.o \ - common/funding_tx.o \ common/status_wiregen.o \ common/hash_u5.o \ common/key_derive.o \ diff --git a/hsmd/hsmd.c b/hsmd/hsmd.c index 1bd52a4dfb04..fe25a542c043 100644 --- a/hsmd/hsmd.c +++ b/hsmd/hsmd.c @@ -27,7 +27,6 @@ #include #include #include -#include #include #include #include diff --git a/lightningd/Makefile b/lightningd/Makefile index 2894fbedca3c..09e41a5363c4 100644 --- a/lightningd/Makefile +++ b/lightningd/Makefile @@ -74,7 +74,6 @@ LIGHTNINGD_COMMON_OBJS := \ common/ecdh_hsmd.o \ common/features.o \ common/fee_states.o \ - common/funding_tx.o \ common/peer_status_wiregen.o \ common/status_wiregen.o \ common/gossip_rcvd_filter.o \ diff --git a/lightningd/opening_control.c b/lightningd/opening_control.c index 023f567e77e1..62897930375b 100644 --- a/lightningd/opening_control.c +++ b/lightningd/opening_control.c @@ -6,7 +6,6 @@ #include #include #include -#include #include #include #include diff --git a/openingd/Makefile b/openingd/Makefile index 585408d609ba..fa076b220545 100644 --- a/openingd/Makefile +++ b/openingd/Makefile @@ -34,7 +34,6 @@ OPENINGD_COMMON_OBJS := \ common/dev_disconnect.o \ common/features.o \ common/fee_states.o \ - common/funding_tx.o \ common/status_wiregen.o \ common/peer_status_wiregen.o \ common/gossip_rcvd_filter.o \ diff --git a/openingd/openingd.c b/openingd/openingd.c index 3520e3b939a9..d365efabe4ba 100644 --- a/openingd/openingd.c +++ b/openingd/openingd.c @@ -22,7 +22,6 @@ #include #include #include -#include #include #include #include From 438953b8f0af6bc891dce33841473458d1e6c943 Mon Sep 17 00:00:00 2001 From: Rusty Russell Date: Fri, 28 Aug 2020 13:26:34 +0930 Subject: [PATCH 23/30] utxo: make reserved_til a u32 not a ptr, now it's compsulory. It's 0 for old dbs, which is the same as "available". Signed-off-by: Rusty Russell --- common/utxo.h | 2 +- wallet/reservation.c | 14 +++++++------- wallet/wallet.c | 35 ++++++++--------------------------- wallet/walletrpc.c | 6 +----- 4 files changed, 17 insertions(+), 40 deletions(-) diff --git a/common/utxo.h b/common/utxo.h index 4dd1c1642aed..8ee7fac25cbc 100644 --- a/common/utxo.h +++ b/common/utxo.h @@ -41,7 +41,7 @@ struct utxo { const u32 *spendheight; /* Block this utxo becomes unreserved, if applicable */ - u32 *reserved_til; + u32 reserved_til; /* The scriptPubkey if it is known */ u8 *scriptPubkey; diff --git a/wallet/reservation.c b/wallet/reservation.c index aa20177bf150..7e489a6b1b55 100644 --- a/wallet/reservation.c +++ b/wallet/reservation.c @@ -12,13 +12,13 @@ #include static bool was_reserved(enum output_status oldstatus, - const u32 *reserved_til, + u32 reserved_til, u32 current_height) { if (oldstatus != output_state_reserved) return false; - return *reserved_til > current_height; + return reserved_til > current_height; } static void json_add_reservestatus(struct json_stream *response, @@ -31,12 +31,12 @@ static void json_add_reservestatus(struct json_stream *response, json_add_txid(response, "txid", &utxo->txid); json_add_u32(response, "vout", utxo->outnum); json_add_bool(response, "was_reserved", - was_reserved(oldstatus, &old_res, current_height)); + was_reserved(oldstatus, old_res, current_height)); json_add_bool(response, "reserved", is_reserved(utxo, current_height)); - if (utxo->reserved_til) + if (is_reserved(utxo, current_height)) json_add_u32(response, "reserved_to_block", - *utxo->reserved_til); + utxo->reserved_til); json_object_end(response); } @@ -52,7 +52,7 @@ static void reserve_and_report(struct json_stream *response, u32 old_res; oldstatus = utxos[i]->status; - old_res = utxos[i]->reserved_til ? *utxos[i]->reserved_til : 0; + old_res = utxos[i]->reserved_til; if (!wallet_reserve_utxo(wallet, utxos[i], @@ -156,7 +156,7 @@ static struct command_result *json_unreserveinputs(struct command *cmd, continue; oldstatus = utxo->status; - old_res = *utxo->reserved_til; + old_res = utxo->reserved_til; wallet_unreserve_utxo(cmd->ld->wallet, utxo, diff --git a/wallet/wallet.c b/wallet/wallet.c index 85b74a64f996..013105780267 100644 --- a/wallet/wallet.c +++ b/wallet/wallet.c @@ -205,8 +205,7 @@ static struct utxo *wallet_stmt2output(const tal_t *ctx, struct db_stmt *stmt) } /* This column can be null if 0.9.1 db or below. */ - utxo->reserved_til = tal(utxo, u32); - *utxo->reserved_til = db_column_int_or_default(stmt, 13, 0); + utxo->reserved_til = db_column_int_or_default(stmt, 13, 0); return utxo; } @@ -434,10 +433,7 @@ static void db_set_utxo(struct db *db, const struct utxo *utxo) db, SQL("UPDATE outputs SET status=?, reserved_til=?" "WHERE prev_out_tx=? AND prev_out_index=?")); db_bind_int(stmt, 0, output_status_in_db(utxo->status)); - if (utxo->reserved_til) - db_bind_int(stmt, 1, *utxo->reserved_til); - else - db_bind_null(stmt, 1); + db_bind_int(stmt, 1, utxo->reserved_til); db_bind_txid(stmt, 2, &utxo->txid); db_bind_int(stmt, 3, utxo->outnum); db_exec_prepared_v2(take(stmt)); @@ -445,11 +441,6 @@ static void db_set_utxo(struct db *db, const struct utxo *utxo) bool wallet_reserve_utxo(struct wallet *w, struct utxo *utxo, u32 current_height) { - u32 reservation_height; - - if (utxo->status == output_state_reserved) - assert(utxo->reserved_til); - switch (utxo->status) { case output_state_spent: return false; @@ -461,15 +452,12 @@ bool wallet_reserve_utxo(struct wallet *w, struct utxo *utxo, u32 current_height } /* We simple increase existing reservations, which DTRT if we unreserve */ - if (utxo->reserved_til - && *utxo->reserved_til >= current_height) - reservation_height = *utxo->reserved_til + RESERVATION_INC; + if (utxo->reserved_til >= current_height) + utxo->reserved_til += RESERVATION_INC; else - reservation_height = current_height + RESERVATION_INC; + utxo->reserved_til = current_height + RESERVATION_INC; utxo->status = output_state_reserved; - tal_free(utxo->reserved_til); - utxo->reserved_til = tal_dup(utxo, u32, &reservation_height); db_set_utxo(w->db, utxo); @@ -478,23 +466,16 @@ bool wallet_reserve_utxo(struct wallet *w, struct utxo *utxo, u32 current_height void wallet_unreserve_utxo(struct wallet *w, struct utxo *utxo, u32 current_height) { - if (utxo->status == output_state_reserved) { - /* FIXME: old code didn't set reserved_til, so fake it here */ - if (!utxo->reserved_til) - utxo->reserved_til = tal_dup(utxo, u32, ¤t_height); - assert(utxo->reserved_til); - } - if (utxo->status != output_state_reserved) fatal("UTXO %s:%u is not reserved", type_to_string(tmpctx, struct bitcoin_txid, &utxo->txid), utxo->outnum); - if (*utxo->reserved_til <= current_height + RESERVATION_INC) { + if (utxo->reserved_til <= current_height + RESERVATION_INC) { utxo->status = output_state_available; - utxo->reserved_til = tal_free(utxo->reserved_til); + utxo->reserved_til = 0; } else - *utxo->reserved_til -= RESERVATION_INC; + utxo->reserved_til -= RESERVATION_INC; db_set_utxo(w->db, utxo); } diff --git a/wallet/walletrpc.c b/wallet/walletrpc.c index c1c69c59b354..ce5f3aa94a7e 100644 --- a/wallet/walletrpc.c +++ b/wallet/walletrpc.c @@ -249,11 +249,7 @@ bool is_reserved(const struct utxo *utxo, u32 current_height) if (utxo->status != output_state_reserved) return false; - /* FIXME: Eventually this will always be set! */ - if (!utxo->reserved_til) - return true; - - return *utxo->reserved_til > current_height; + return utxo->reserved_til > current_height; } From e3219d3aa0b0289f32c32172cb405ba4c574cd08 Mon Sep 17 00:00:00 2001 From: Rusty Russell Date: Fri, 28 Aug 2020 13:26:34 +0930 Subject: [PATCH 24/30] utxo: expose is_reserved, make enum constants upper case. Signed-off-by: Rusty Russell --- common/utxo.h | 25 ++++++++++++++++++++++- wallet/reservation.c | 16 +++++++-------- wallet/test/run-wallet.c | 16 +++++++-------- wallet/wallet.c | 44 ++++++++++++++++++++-------------------- wallet/wallet.h | 28 +++++++------------------ wallet/walletrpc.c | 23 +++++++-------------- 6 files changed, 76 insertions(+), 76 deletions(-) diff --git a/common/utxo.h b/common/utxo.h index 8ee7fac25cbc..f779328ea39b 100644 --- a/common/utxo.h +++ b/common/utxo.h @@ -22,13 +22,27 @@ struct unilateral_close_info { struct pubkey *commitment_point; }; +/* Possible states for tracked outputs in the database. Not sure yet + * whether we really want to have reservations reflected in the + * database, it would simplify queries at the cost of some IO ops */ +/* /!\ This is a DB ENUM, please do not change the numbering of any + * already defined elements (adding is ok) /!\ */ +enum output_status { + OUTPUT_STATE_AVAILABLE = 0, + OUTPUT_STATE_RESERVED = 1, + OUTPUT_STATE_SPENT = 2, + /* Special status used to express that we don't care in + * queries */ + OUTPUT_STATE_ANY = 255 +}; + struct utxo { struct bitcoin_txid txid; u32 outnum; struct amount_sat amount; u32 keyindex; bool is_p2sh; - u8 status; + enum output_status status; /* Optional unilateral close information, NULL if this is just * a HD key */ @@ -47,6 +61,15 @@ struct utxo { u8 *scriptPubkey; }; +/* We lazy-evaluate whether a utxo is really still reserved. */ +static inline bool utxo_is_reserved(const struct utxo *utxo, u32 current_height) +{ + if (utxo->status != OUTPUT_STATE_RESERVED) + return false; + + return utxo->reserved_til > current_height; +} + void towire_utxo(u8 **pptr, const struct utxo *utxo); struct utxo *fromwire_utxo(const tal_t *ctx, const u8 **ptr, size_t *max); diff --git a/wallet/reservation.c b/wallet/reservation.c index 7e489a6b1b55..b1df871f0773 100644 --- a/wallet/reservation.c +++ b/wallet/reservation.c @@ -15,7 +15,7 @@ static bool was_reserved(enum output_status oldstatus, u32 reserved_til, u32 current_height) { - if (oldstatus != output_state_reserved) + if (oldstatus != OUTPUT_STATE_RESERVED) return false; return reserved_til > current_height; @@ -33,8 +33,8 @@ static void json_add_reservestatus(struct json_stream *response, json_add_bool(response, "was_reserved", was_reserved(oldstatus, old_res, current_height)); json_add_bool(response, "reserved", - is_reserved(utxo, current_height)); - if (is_reserved(utxo, current_height)) + utxo_is_reserved(utxo, current_height)); + if (utxo_is_reserved(utxo, current_height)) json_add_u32(response, "reserved_to_block", utxo->reserved_til); json_object_end(response); @@ -96,7 +96,7 @@ static struct command_result *json_reserveinputs(struct command *cmd, &txid, psbt->tx->inputs[i].index); if (!utxo) continue; - if (*exclusive && is_reserved(utxo, current_height)) { + if (*exclusive && utxo_is_reserved(utxo, current_height)) { return command_fail(cmd, JSONRPC2_INVALID_PARAMS, "%s:%u already reserved", type_to_string(tmpctx, @@ -104,7 +104,7 @@ static struct command_result *json_reserveinputs(struct command *cmd, &utxo->txid), utxo->outnum); } - if (utxo->status == output_state_spent) + if (utxo->status == OUTPUT_STATE_SPENT) return command_fail(cmd, JSONRPC2_INVALID_PARAMS, "%s:%u already spent", type_to_string(tmpctx, @@ -152,7 +152,7 @@ static struct command_result *json_unreserveinputs(struct command *cmd, wally_tx_input_get_txid(&psbt->tx->inputs[i], &txid); utxo = wallet_utxo_get(cmd, cmd->ld->wallet, &txid, psbt->tx->inputs[i].index); - if (!utxo || utxo->status != output_state_reserved) + if (!utxo || utxo->status != OUTPUT_STATE_RESERVED) continue; oldstatus = utxo->status; @@ -477,7 +477,7 @@ static struct command_result *param_txout(struct command *cmd, &txid), outnum); } - if (utxo->status == output_state_spent) { + if (utxo->status == OUTPUT_STATE_SPENT) { return command_fail(cmd, JSONRPC2_INVALID_PARAMS, "Already spent UTXO %s:%u", type_to_string(tmpctx, @@ -527,7 +527,7 @@ static struct command_result *json_utxopsbt(struct command *cmd, for (size_t i = 0; i < tal_count(utxos); i++) { const struct utxo *utxo = utxos[i]; - if (!*reserved_ok && is_reserved(utxo, current_height)) + if (!*reserved_ok && utxo_is_reserved(utxo, current_height)) return command_fail(cmd, JSONRPC2_INVALID_PARAMS, "UTXO %s:%u already reserved", type_to_string(tmpctx, diff --git a/wallet/test/run-wallet.c b/wallet/test/run-wallet.c index 652da3ea6631..dec6a47e079c 100644 --- a/wallet/test/run-wallet.c +++ b/wallet/test/run-wallet.c @@ -961,26 +961,26 @@ static bool test_wallet_outputs(struct lightningd *ld, const tal_t *ctx) /* Attempt to reserve the utxo */ CHECK_MSG(wallet_update_output_status(w, &u.txid, u.outnum, - output_state_available, - output_state_reserved), + OUTPUT_STATE_AVAILABLE, + OUTPUT_STATE_RESERVED), "could not reserve available output"); /* Reserving twice should fail */ CHECK_MSG(!wallet_update_output_status(w, &u.txid, u.outnum, - output_state_available, - output_state_reserved), + OUTPUT_STATE_AVAILABLE, + OUTPUT_STATE_RESERVED), "could reserve already reserved output"); /* Un-reserving should work */ CHECK_MSG(wallet_update_output_status(w, &u.txid, u.outnum, - output_state_reserved, - output_state_available), + OUTPUT_STATE_RESERVED, + OUTPUT_STATE_AVAILABLE), "could not unreserve reserved output"); /* Switching from any to something else */ CHECK_MSG(wallet_update_output_status(w, &u.txid, u.outnum, - output_state_any, - output_state_spent), + OUTPUT_STATE_ANY, + OUTPUT_STATE_SPENT), "could not change output state ignoring oldstate"); /* Attempt to save an UTXO with close_info set, no commitment_point */ diff --git a/wallet/wallet.c b/wallet/wallet.c index 013105780267..8e13bb530eb5 100644 --- a/wallet/wallet.c +++ b/wallet/wallet.c @@ -34,7 +34,7 @@ static void outpointfilters_init(struct wallet *w) { struct db_stmt *stmt; - struct utxo **utxos = wallet_get_utxos(NULL, w, output_state_any); + struct utxo **utxos = wallet_get_utxos(NULL, w, OUTPUT_STATE_ANY); struct bitcoin_txid txid; u32 outnum; @@ -121,7 +121,7 @@ static bool wallet_add_utxo(struct wallet *w, struct utxo *utxo, db_bind_int(stmt, 1, utxo->outnum); db_bind_amount_sat(stmt, 2, &utxo->amount); db_bind_int(stmt, 3, wallet_output_type_in_db(type)); - db_bind_int(stmt, 4, output_state_available); + db_bind_int(stmt, 4, OUTPUT_STATE_AVAILABLE); db_bind_int(stmt, 5, utxo->keyindex); if (utxo->close_info) { db_bind_u64(stmt, 6, utxo->close_info->channel_id); @@ -217,7 +217,7 @@ bool wallet_update_output_status(struct wallet *w, { struct db_stmt *stmt; size_t changes; - if (oldstatus != output_state_any) { + if (oldstatus != OUTPUT_STATE_ANY) { stmt = db_prepare_v2( w->db, SQL("UPDATE outputs SET status=? WHERE status=? AND " "prev_out_tx=? AND prev_out_index=?")); @@ -245,7 +245,7 @@ struct utxo **wallet_get_utxos(const tal_t *ctx, struct wallet *w, const enum ou int i; struct db_stmt *stmt; - if (state == output_state_any) { + if (state == OUTPUT_STATE_ANY) { stmt = db_prepare_v2(w->db, SQL("SELECT" " prev_out_tx" ", prev_out_index" @@ -378,8 +378,8 @@ bool wallet_unreserve_output(struct wallet *w, const u32 outnum) { return wallet_update_output_status(w, txid, outnum, - output_state_reserved, - output_state_available); + OUTPUT_STATE_RESERVED, + OUTPUT_STATE_AVAILABLE); } /** @@ -388,8 +388,8 @@ bool wallet_unreserve_output(struct wallet *w, static void unreserve_utxo(struct wallet *w, const struct utxo *unres) { if (!wallet_update_output_status(w, &unres->txid, unres->outnum, - output_state_reserved, - output_state_available)) { + OUTPUT_STATE_RESERVED, + OUTPUT_STATE_AVAILABLE)) { fatal("Unable to unreserve output"); } } @@ -414,7 +414,7 @@ void wallet_confirm_utxos(struct wallet *w, const struct utxo **utxos) for (size_t i = 0; i < tal_count(utxos); i++) { if (!wallet_update_output_status( w, &utxos[i]->txid, utxos[i]->outnum, - output_state_reserved, output_state_spent)) { + OUTPUT_STATE_RESERVED, OUTPUT_STATE_SPENT)) { fatal("Unable to mark output as spent"); } } @@ -424,7 +424,7 @@ static void db_set_utxo(struct db *db, const struct utxo *utxo) { struct db_stmt *stmt; - if (utxo->status == output_state_reserved) + if (utxo->status == OUTPUT_STATE_RESERVED) assert(utxo->reserved_til); else assert(!utxo->reserved_til); @@ -442,12 +442,12 @@ static void db_set_utxo(struct db *db, const struct utxo *utxo) bool wallet_reserve_utxo(struct wallet *w, struct utxo *utxo, u32 current_height) { switch (utxo->status) { - case output_state_spent: + case OUTPUT_STATE_SPENT: return false; - case output_state_available: - case output_state_reserved: + case OUTPUT_STATE_AVAILABLE: + case OUTPUT_STATE_RESERVED: break; - case output_state_any: + case OUTPUT_STATE_ANY: abort(); } @@ -457,7 +457,7 @@ bool wallet_reserve_utxo(struct wallet *w, struct utxo *utxo, u32 current_height else utxo->reserved_til = current_height + RESERVATION_INC; - utxo->status = output_state_reserved; + utxo->status = OUTPUT_STATE_RESERVED; db_set_utxo(w->db, utxo); @@ -466,13 +466,13 @@ bool wallet_reserve_utxo(struct wallet *w, struct utxo *utxo, u32 current_height void wallet_unreserve_utxo(struct wallet *w, struct utxo *utxo, u32 current_height) { - if (utxo->status != output_state_reserved) + if (utxo->status != OUTPUT_STATE_RESERVED) fatal("UTXO %s:%u is not reserved", type_to_string(tmpctx, struct bitcoin_txid, &utxo->txid), utxo->outnum); if (utxo->reserved_til <= current_height + RESERVATION_INC) { - utxo->status = output_state_available; + utxo->status = OUTPUT_STATE_AVAILABLE; utxo->reserved_til = 0; } else utxo->reserved_til -= RESERVATION_INC; @@ -534,8 +534,8 @@ struct utxo *wallet_find_utxo(const tal_t *ctx, struct wallet *w, " WHERE status = ?" " OR (status = ? AND reserved_til <= ?)" "ORDER BY RANDOM();")); - db_bind_int(stmt, 0, output_status_in_db(output_state_available)); - db_bind_int(stmt, 1, output_status_in_db(output_state_reserved)); + db_bind_int(stmt, 0, output_status_in_db(OUTPUT_STATE_AVAILABLE)); + db_bind_int(stmt, 1, output_status_in_db(OUTPUT_STATE_RESERVED)); db_bind_u64(stmt, 2, current_blockheight); /* FIXME: Use feerate + estimate of input cost to establish @@ -599,7 +599,7 @@ bool wallet_add_onchaind_utxo(struct wallet *w, db_bind_int(stmt, 1, outnum); db_bind_amount_sat(stmt, 2, &amount); db_bind_int(stmt, 3, wallet_output_type_in_db(p2wpkh)); - db_bind_int(stmt, 4, output_state_available); + db_bind_int(stmt, 4, OUTPUT_STATE_AVAILABLE); db_bind_int(stmt, 5, 0); db_bind_u64(stmt, 6, channel->dbid); db_bind_node_id(stmt, 7, &channel->peer->id); @@ -1734,7 +1734,7 @@ int wallet_extract_owned_outputs(struct wallet *w, const struct wally_tx *wtx, utxo->keyindex = index; utxo->is_p2sh = is_p2sh; utxo->amount = amount_asset_to_sat(&asset); - utxo->status = output_state_available; + utxo->status = OUTPUT_STATE_AVAILABLE; wally_txid(wtx, &utxo->txid); utxo->outnum = output; utxo->close_info = NULL; @@ -3019,7 +3019,7 @@ wallet_outpoint_spend(struct wallet *w, const tal_t *ctx, const u32 blockheight, " AND prev_out_index = ?")); db_bind_int(stmt, 0, blockheight); - db_bind_int(stmt, 1, output_status_in_db(output_state_spent)); + db_bind_int(stmt, 1, output_status_in_db(OUTPUT_STATE_SPENT)); db_bind_sha256d(stmt, 2, &txid->shad); db_bind_int(stmt, 3, outnum); diff --git a/wallet/wallet.h b/wallet/wallet.h index 73b2347e2cb2..8220408a68c6 100644 --- a/wallet/wallet.h +++ b/wallet/wallet.h @@ -52,34 +52,20 @@ struct wallet { u64 keyscan_gap; }; -/* Possible states for tracked outputs in the database. Not sure yet - * whether we really want to have reservations reflected in the - * database, it would simplify queries at the cost of some IO ops */ -/* /!\ This is a DB ENUM, please do not change the numbering of any - * already defined elements (adding is ok) /!\ */ -enum output_status { - output_state_available= 0, - output_state_reserved = 1, - output_state_spent = 2, - /* Special status used to express that we don't care in - * queries */ - output_state_any = 255 -}; - static inline enum output_status output_status_in_db(enum output_status s) { switch (s) { - case output_state_available: - BUILD_ASSERT(output_state_available == 0); + case OUTPUT_STATE_AVAILABLE: + BUILD_ASSERT(OUTPUT_STATE_AVAILABLE == 0); return s; - case output_state_reserved: - BUILD_ASSERT(output_state_reserved == 1); + case OUTPUT_STATE_RESERVED: + BUILD_ASSERT(OUTPUT_STATE_RESERVED == 1); return s; - case output_state_spent: - BUILD_ASSERT(output_state_spent == 2); + case OUTPUT_STATE_SPENT: + BUILD_ASSERT(OUTPUT_STATE_SPENT == 2); return s; /* This one doesn't go into db */ - case output_state_any: + case OUTPUT_STATE_ANY: break; } fatal("%s: %u is invalid", __func__, s); diff --git a/wallet/walletrpc.c b/wallet/walletrpc.c index ce5f3aa94a7e..26f8073d3bcd 100644 --- a/wallet/walletrpc.c +++ b/wallet/walletrpc.c @@ -244,15 +244,6 @@ static const struct json_command listaddrs_command = { }; AUTODATA(json_command, &listaddrs_command); -bool is_reserved(const struct utxo *utxo, u32 current_height) -{ - if (utxo->status != output_state_reserved) - return false; - - return utxo->reserved_til > current_height; -} - - static void json_add_utxo(struct json_stream *response, const char *fieldname, struct wallet *wallet, @@ -297,8 +288,8 @@ static void json_add_utxo(struct json_stream *response, json_add_string(response, "status", "unconfirmed"); json_add_bool(response, "reserved", - is_reserved(utxo, - get_block_height(wallet->ld->topology))); + utxo_is_reserved(utxo, + get_block_height(wallet->ld->topology))); json_object_end(response); } @@ -324,8 +315,8 @@ static struct command_result *json_listfunds(struct command *cmd, response = json_stream_success(cmd); - utxos = wallet_get_utxos(cmd, cmd->ld->wallet, output_state_available); - reserved_utxos = wallet_get_utxos(cmd, cmd->ld->wallet, output_state_reserved); + utxos = wallet_get_utxos(cmd, cmd->ld->wallet, OUTPUT_STATE_AVAILABLE); + reserved_utxos = wallet_get_utxos(cmd, cmd->ld->wallet, OUTPUT_STATE_RESERVED); json_array_start(response, "outputs"); json_add_utxos(response, cmd->ld->wallet, utxos); json_add_utxos(response, cmd->ld->wallet, reserved_utxos); @@ -391,7 +382,7 @@ static void process_utxo_result(struct bitcoind *bitcoind, struct json_stream *response = rescan->response; struct utxo *u = rescan->utxos[0]; enum output_status newstate = - txout == NULL ? output_state_spent : output_state_available; + txout == NULL ? OUTPUT_STATE_SPENT : OUTPUT_STATE_AVAILABLE; json_object_start(rescan->response, NULL); json_add_txid(response, "txid", &u->txid); @@ -432,7 +423,7 @@ static struct command_result *json_dev_rescan_outputs(struct command *cmd, /* Open the outputs structure so we can incrementally add results */ json_array_start(rescan->response, "outputs"); - rescan->utxos = wallet_get_utxos(rescan, cmd->ld->wallet, output_state_any); + rescan->utxos = wallet_get_utxos(rescan, cmd->ld->wallet, OUTPUT_STATE_ANY); if (tal_count(rescan->utxos) == 0) { json_array_end(rescan->response); return command_success(cmd, rescan->response); @@ -654,7 +645,7 @@ static struct command_result *match_psbt_inputs_to_utxos(struct command *cmd, } /* Oops we haven't reserved this utxo yet! */ - if (!is_reserved(utxo, get_block_height(cmd->ld->topology))) + if (!utxo_is_reserved(utxo, get_block_height(cmd->ld->topology))) return command_fail(cmd, LIGHTNINGD, "Aborting PSBT signing. UTXO %s:%u is not reserved", type_to_string(tmpctx, struct bitcoin_txid, From 73832d148853d2683f223d6720f4b0e389039506 Mon Sep 17 00:00:00 2001 From: Rusty Russell Date: Fri, 28 Aug 2020 13:26:34 +0930 Subject: [PATCH 25/30] wellet: use create_psbt and psbt_append_input instead of constructing via bitcoin_tx. Suggested-by: @niftynei Signed-off-by: Rusty Russell --- wallet/reservation.c | 18 ++++++------------ 1 file changed, 6 insertions(+), 12 deletions(-) diff --git a/wallet/reservation.c b/wallet/reservation.c index b1df871f0773..3f6dff6541b2 100644 --- a/wallet/reservation.c +++ b/wallet/reservation.c @@ -219,10 +219,9 @@ static struct wally_psbt *psbt_using_utxos(const tal_t *ctx, { struct pubkey key; u8 *scriptSig, *scriptPubkey, *redeemscript; - struct bitcoin_tx *tx; + struct wally_psbt *psbt; - /* FIXME: Currently the easiest way to get a PSBT is via a tx */ - tx = bitcoin_tx(ctx, chainparams, tal_count(utxos), 0, nlocktime); + psbt = create_psbt(ctx, tal_count(utxos), 0, nlocktime); for (size_t i = 0; i < tal_count(utxos); i++) { u32 this_nsequence; @@ -256,17 +255,12 @@ static struct wally_psbt *psbt_using_utxos(const tal_t *ctx, else this_nsequence = nsequence; - bitcoin_tx_add_input(tx, &utxos[i]->txid, utxos[i]->outnum, - this_nsequence, scriptSig, utxos[i]->amount, - scriptPubkey, NULL); - - /* Add redeemscript to the PSBT input */ - if (redeemscript) - psbt_input_set_redeemscript(tx->psbt, i, redeemscript); - + psbt_append_input(psbt, &utxos[i]->txid, utxos[i]->outnum, + this_nsequence, scriptSig, utxos[i]->amount, + scriptPubkey, NULL, redeemscript); } - return tx->psbt; + return psbt; } static struct command_result *finish_psbt(struct command *cmd, From c9687ab6ce338d38029a275caa10df127b557d30 Mon Sep 17 00:00:00 2001 From: Vincenzo Palazzo Date: Mon, 7 Sep 2020 13:00:36 +0200 Subject: [PATCH 26/30] Added proxy information to config information for plugins Changelog-Added: Plugins: Proxy information now provided in init.configuration." Signed-off-by: Vincenzo Palazzo --- doc/PLUGINS.md | 8 ++++++++ lightningd/plugin.c | 6 ++++++ 2 files changed, 14 insertions(+) diff --git a/doc/PLUGINS.md b/doc/PLUGINS.md index 347abf882111..88cf0a3c9f31 100644 --- a/doc/PLUGINS.md +++ b/doc/PLUGINS.md @@ -214,6 +214,14 @@ simple JSON object containing the options: "channel": "", "invoice": "028200" } + "startup": true, + "proxy": { + "type": "ipv4", + "address": "127.0.0.1", + "port": 9050 + } + "torv3-enabled": true, + "use_proxy_always": false } } ``` diff --git a/lightningd/plugin.c b/lightningd/plugin.c index 1f696a2eca3c..43ecf48aa8dd 100644 --- a/lightningd/plugin.c +++ b/lightningd/plugin.c @@ -6,6 +6,7 @@ #include #include #include +#include #include #include #include @@ -1452,6 +1453,11 @@ plugin_populate_init_request(struct plugin *plugin, struct jsonrpc_request *req) json_add_string(req->stream, "rpc-file", ld->rpc_filename); json_add_bool(req->stream, "startup", plugin->plugins->startup); json_add_string(req->stream, "network", chainparams->network_name); + if (ld->proxyaddr) { + json_add_address(req->stream, "proxy", ld->proxyaddr); + json_add_bool(req->stream, "torv3-enabled", ld->config.use_v3_autotor); + json_add_bool(req->stream, "use_proxy_always", ld->use_proxy_always); + } json_object_start(req->stream, "feature_set"); for (enum feature_place fp = 0; fp < NUM_FEATURE_PLACE; fp++) { if (feature_place_names[fp]) { From 348f5c50dacb3114d34f69af20de0f6cba2c7143 Mon Sep 17 00:00:00 2001 From: Rusty Russell Date: Tue, 8 Sep 2020 10:43:18 +0930 Subject: [PATCH 27/30] Makefile: fix up sqlgen and docgen rules. Omitted in commit d8e8426b52d9dd097c361e3cf405edbdfc20cffe Signed-off-by: Rusty Russell --- doc/Makefile | 2 +- wallet/Makefile | 4 +- wallet/db_postgres_sqlgen.c | 8 +- wallet/db_sqlite3_sqlgen.c | 8 +- wallet/statements_gettextgen.po | 214 ++++++++++++++++---------------- 5 files changed, 118 insertions(+), 118 deletions(-) diff --git a/doc/Makefile b/doc/Makefile index 9d611cf12fc2..1bcbbd353496 100644 --- a/doc/Makefile +++ b/doc/Makefile @@ -58,7 +58,7 @@ MANPAGES := doc/lightning-cli.1 \ doc-all: $(MANPAGES) doc/index.rst $(MANPAGES): doc/%: doc/%.md - @if $(call SHA256STAMP_CHANGED,); then $(call VERBOSE, "mrkd $<", mrkd $< $@ && $(call SHA256STAMP,,\")); else touch $@; fi + @if $(call SHA256STAMP_CHANGED); then $(call VERBOSE, "mrkd $<", mrkd $< $@ && $(call SHA256STAMP,\")); else touch $@; fi $(MANPAGES): FORCE diff --git a/wallet/Makefile b/wallet/Makefile index d8d97a3cddbb..efe9c3528f4f 100644 --- a/wallet/Makefile +++ b/wallet/Makefile @@ -37,10 +37,10 @@ SQL_FILES := \ wallet/test/run-wallet.c \ wallet/statements_gettextgen.po: $(SQL_FILES) FORCE - @if $(call SHA256STAMP_CHANGED,); then $(call VERBOSE,"xgettext $@",xgettext -kNAMED_SQL -kSQL --add-location --no-wrap --omit-header -o $@ $(SQL_FILES) && $(call SHA256STAMP,,# )); fi + @if $(call SHA256STAMP_CHANGED); then $(call VERBOSE,"xgettext $@",xgettext -kNAMED_SQL -kSQL --add-location --no-wrap --omit-header -o $@ $(SQL_FILES) && $(call SHA256STAMP,# )); fi wallet/db_%_sqlgen.c: wallet/statements_gettextgen.po devtools/sql-rewrite.py FORCE - @if $(call SHA256STAMP_CHANGED,); then $(call VERBOSE,"sql-rewrite $@",devtools/sql-rewrite.py wallet/statements_gettextgen.po $* > $@ && $(call SHA256STAMP,,//)); fi + @if $(call SHA256STAMP_CHANGED); then $(call VERBOSE,"sql-rewrite $@",devtools/sql-rewrite.py wallet/statements_gettextgen.po $* > $@ && $(call SHA256STAMP,//)); fi maintainer-clean: wallet-maintainer-clean wallet-maintainer-clean: diff --git a/wallet/db_postgres_sqlgen.c b/wallet/db_postgres_sqlgen.c index 183a99d72182..2b3aaa477329 100644 --- a/wallet/db_postgres_sqlgen.c +++ b/wallet/db_postgres_sqlgen.c @@ -1455,9 +1455,9 @@ struct db_query db_postgres_queries[] = { .readonly = false, }, { - .name = "UPDATE outputs SET spend_height = ? WHERE prev_out_tx = ? AND prev_out_index = ?", - .query = "UPDATE outputs SET spend_height = $1 WHERE prev_out_tx = $2 AND prev_out_index = $3", - .placeholders = 3, + .name = "UPDATE outputs SET spend_height = ?, status = ? WHERE prev_out_tx = ? AND prev_out_index = ?", + .query = "UPDATE outputs SET spend_height = $1, status = $2 WHERE prev_out_tx = $3 AND prev_out_index = $4", + .placeholders = 4, .readonly = false, }, { @@ -1642,4 +1642,4 @@ struct db_query db_postgres_queries[] = { #endif /* LIGHTNINGD_WALLET_GEN_DB_POSTGRES */ -// SHA256STAMP:70da3f4fe74de7abdc4cfd910e203ecd4ded34213384fcb7027a067b4cb56590 +// SHA256STAMP:a4a3f2d9ee677312ff2586432ffae389eedb72f1bf34d36bfb564d37889f8eeb diff --git a/wallet/db_sqlite3_sqlgen.c b/wallet/db_sqlite3_sqlgen.c index d833fcadd66d..b5488addfedf 100644 --- a/wallet/db_sqlite3_sqlgen.c +++ b/wallet/db_sqlite3_sqlgen.c @@ -1455,9 +1455,9 @@ struct db_query db_sqlite3_queries[] = { .readonly = false, }, { - .name = "UPDATE outputs SET spend_height = ? WHERE prev_out_tx = ? AND prev_out_index = ?", - .query = "UPDATE outputs SET spend_height = ? WHERE prev_out_tx = ? AND prev_out_index = ?", - .placeholders = 3, + .name = "UPDATE outputs SET spend_height = ?, status = ? WHERE prev_out_tx = ? AND prev_out_index = ?", + .query = "UPDATE outputs SET spend_height = ?, status = ? WHERE prev_out_tx = ? AND prev_out_index = ?", + .placeholders = 4, .readonly = false, }, { @@ -1642,4 +1642,4 @@ struct db_query db_sqlite3_queries[] = { #endif /* LIGHTNINGD_WALLET_GEN_DB_SQLITE3 */ -// SHA256STAMP:70da3f4fe74de7abdc4cfd910e203ecd4ded34213384fcb7027a067b4cb56590 +// SHA256STAMP:a4a3f2d9ee677312ff2586432ffae389eedb72f1bf34d36bfb564d37889f8eeb diff --git a/wallet/statements_gettextgen.po b/wallet/statements_gettextgen.po index adf75c2fe22a..0798bed35db8 100644 --- a/wallet/statements_gettextgen.po +++ b/wallet/statements_gettextgen.po @@ -654,419 +654,419 @@ msgstr "" msgid "SELECT txid, outnum FROM utxoset WHERE spendheight is NULL" msgstr "" -#: wallet/wallet.c:92 wallet/wallet.c:595 +#: wallet/wallet.c:91 wallet/wallet.c:569 msgid "SELECT * from outputs WHERE prev_out_tx=? AND prev_out_index=?" msgstr "" -#: wallet/wallet.c:106 wallet/wallet.c:609 +#: wallet/wallet.c:105 wallet/wallet.c:583 msgid "INSERT INTO outputs ( prev_out_tx, prev_out_index, value, type, status, keyindex, channel_id, peer_id, commitment_point, option_anchor_outputs, confirmation_height, spend_height, scriptpubkey) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?);" msgstr "" -#: wallet/wallet.c:227 +#: wallet/wallet.c:222 msgid "UPDATE outputs SET status=? WHERE status=? AND prev_out_tx=? AND prev_out_index=?" msgstr "" -#: wallet/wallet.c:235 +#: wallet/wallet.c:230 msgid "UPDATE outputs SET status=? WHERE prev_out_tx=? AND prev_out_index=?" msgstr "" -#: wallet/wallet.c:254 +#: wallet/wallet.c:249 msgid "SELECT prev_out_tx, prev_out_index, value, type, status, keyindex, channel_id, peer_id, commitment_point, option_anchor_outputs, confirmation_height, spend_height, scriptpubkey , reserved_til FROM outputs" msgstr "" -#: wallet/wallet.c:271 +#: wallet/wallet.c:266 msgid "SELECT prev_out_tx, prev_out_index, value, type, status, keyindex, channel_id, peer_id, commitment_point, option_anchor_outputs, confirmation_height, spend_height, scriptpubkey , reserved_til FROM outputs WHERE status= ? " msgstr "" -#: wallet/wallet.c:309 +#: wallet/wallet.c:304 msgid "SELECT prev_out_tx, prev_out_index, value, type, status, keyindex, channel_id, peer_id, commitment_point, option_anchor_outputs, confirmation_height, spend_height, scriptpubkey, reserved_til FROM outputs WHERE channel_id IS NOT NULL AND confirmation_height IS NULL" msgstr "" -#: wallet/wallet.c:346 +#: wallet/wallet.c:341 msgid "SELECT prev_out_tx, prev_out_index, value, type, status, keyindex, channel_id, peer_id, commitment_point, option_anchor_outputs, confirmation_height, spend_height, scriptpubkey, reserved_til FROM outputs WHERE prev_out_tx = ? AND prev_out_index = ?" msgstr "" -#: wallet/wallet.c:438 +#: wallet/wallet.c:433 msgid "UPDATE outputs SET status=?, reserved_til=?WHERE prev_out_tx=? AND prev_out_index=?" msgstr "" -#: wallet/wallet.c:544 +#: wallet/wallet.c:518 msgid "SELECT prev_out_tx, prev_out_index, value, type, status, keyindex, channel_id, peer_id, commitment_point, option_anchor_outputs, confirmation_height, spend_height, scriptpubkey , reserved_til FROM outputs WHERE status = ? OR (status = ? AND reserved_til <= ?)ORDER BY RANDOM();" msgstr "" -#: wallet/wallet.c:882 +#: wallet/wallet.c:687 msgid "INSERT INTO shachains (min_index, num_valid) VALUES (?, 0);" msgstr "" -#: wallet/wallet.c:926 +#: wallet/wallet.c:731 msgid "UPDATE shachains SET num_valid=?, min_index=? WHERE id=?" msgstr "" -#: wallet/wallet.c:933 +#: wallet/wallet.c:738 msgid "UPDATE shachain_known SET idx=?, hash=? WHERE shachain_id=? AND pos=?" msgstr "" -#: wallet/wallet.c:945 +#: wallet/wallet.c:750 msgid "INSERT INTO shachain_known (shachain_id, pos, idx, hash) VALUES (?, ?, ?, ?);" msgstr "" -#: wallet/wallet.c:967 +#: wallet/wallet.c:772 msgid "SELECT min_index, num_valid FROM shachains WHERE id=?" msgstr "" -#: wallet/wallet.c:982 +#: wallet/wallet.c:787 msgid "SELECT idx, hash, pos FROM shachain_known WHERE shachain_id=?" msgstr "" -#: wallet/wallet.c:1005 +#: wallet/wallet.c:810 msgid "SELECT id, node_id, address FROM peers WHERE id=?;" msgstr "" -#: wallet/wallet.c:1036 +#: wallet/wallet.c:841 msgid "SELECT signature FROM htlc_sigs WHERE channelid = ?" msgstr "" -#: wallet/wallet.c:1070 +#: wallet/wallet.c:875 msgid "SELECT remote_ann_node_sig, remote_ann_bitcoin_sig FROM channels WHERE id = ?" msgstr "" -#: wallet/wallet.c:1114 +#: wallet/wallet.c:919 msgid "SELECT hstate, feerate_per_kw FROM channel_feerates WHERE channel_id = ?" msgstr "" -#: wallet/wallet.c:1319 +#: wallet/wallet.c:1124 msgid "SELECT id FROM channels ORDER BY id DESC LIMIT 1;" msgstr "" -#: wallet/wallet.c:1336 +#: wallet/wallet.c:1141 msgid "SELECT id, peer_id, short_channel_id, channel_config_local, channel_config_remote, state, funder, channel_flags, minimum_depth, next_index_local, next_index_remote, next_htlc_id, funding_tx_id, funding_tx_outnum, funding_satoshi, our_funding_satoshi, funding_locked_remote, push_msatoshi, msatoshi_local, fundingkey_remote, revocation_basepoint_remote, payment_basepoint_remote, htlc_basepoint_remote, delayed_payment_basepoint_remote, per_commit_remote, old_per_commit_remote, local_feerate_per_kw, remote_feerate_per_kw, shachain_remote_id, shutdown_scriptpubkey_remote, shutdown_keyidx_local, last_sent_commit_state, last_sent_commit_id, last_tx, last_sig, last_was_revoke, first_blocknum, min_possible_feerate, max_possible_feerate, msatoshi_to_us_min, msatoshi_to_us_max, future_per_commitment_point, last_sent_commit, feerate_base, feerate_ppm, remote_upfront_shutdown_script, option_static_remotekey, option_anchor_outputs, shutdown_scriptpubkey_local FROM channels WHERE state < ?;" msgstr "" -#: wallet/wallet.c:1424 +#: wallet/wallet.c:1229 msgid "UPDATE channels SET in_payments_offered = COALESCE(in_payments_offered, 0) + 1 , in_msatoshi_offered = COALESCE(in_msatoshi_offered, 0) + ? WHERE id = ?;" msgstr "" -#: wallet/wallet.c:1429 +#: wallet/wallet.c:1234 msgid "UPDATE channels SET in_payments_fulfilled = COALESCE(in_payments_fulfilled, 0) + 1 , in_msatoshi_fulfilled = COALESCE(in_msatoshi_fulfilled, 0) + ? WHERE id = ?;" msgstr "" -#: wallet/wallet.c:1434 +#: wallet/wallet.c:1239 msgid "UPDATE channels SET out_payments_offered = COALESCE(out_payments_offered, 0) + 1 , out_msatoshi_offered = COALESCE(out_msatoshi_offered, 0) + ? WHERE id = ?;" msgstr "" -#: wallet/wallet.c:1439 +#: wallet/wallet.c:1244 msgid "UPDATE channels SET out_payments_fulfilled = COALESCE(out_payments_fulfilled, 0) + 1 , out_msatoshi_fulfilled = COALESCE(out_msatoshi_fulfilled, 0) + ? WHERE id = ?;" msgstr "" -#: wallet/wallet.c:1481 +#: wallet/wallet.c:1286 msgid "SELECT in_payments_offered, in_payments_fulfilled, in_msatoshi_offered, in_msatoshi_fulfilled, out_payments_offered, out_payments_fulfilled, out_msatoshi_offered, out_msatoshi_fulfilled FROM channels WHERE id = ?" msgstr "" -#: wallet/wallet.c:1510 +#: wallet/wallet.c:1315 msgid "SELECT MIN(height), MAX(height) FROM blocks;" msgstr "" -#: wallet/wallet.c:1532 +#: wallet/wallet.c:1337 msgid "INSERT INTO channel_configs DEFAULT VALUES;" msgstr "" -#: wallet/wallet.c:1544 +#: wallet/wallet.c:1349 msgid "UPDATE channel_configs SET dust_limit_satoshis=?, max_htlc_value_in_flight_msat=?, channel_reserve_satoshis=?, htlc_minimum_msat=?, to_self_delay=?, max_accepted_htlcs=? WHERE id=?;" msgstr "" -#: wallet/wallet.c:1568 +#: wallet/wallet.c:1373 msgid "SELECT id, dust_limit_satoshis, max_htlc_value_in_flight_msat, channel_reserve_satoshis, htlc_minimum_msat, to_self_delay, max_accepted_htlcs FROM channel_configs WHERE id= ? ;" msgstr "" -#: wallet/wallet.c:1602 +#: wallet/wallet.c:1407 msgid "UPDATE channels SET remote_ann_node_sig=?, remote_ann_bitcoin_sig=? WHERE id=?" msgstr "" -#: wallet/wallet.c:1621 +#: wallet/wallet.c:1426 msgid "UPDATE channels SET shachain_remote_id=?, short_channel_id=?, state=?, funder=?, channel_flags=?, minimum_depth=?, next_index_local=?, next_index_remote=?, next_htlc_id=?, funding_tx_id=?, funding_tx_outnum=?, funding_satoshi=?, our_funding_satoshi=?, funding_locked_remote=?, push_msatoshi=?, msatoshi_local=?, shutdown_scriptpubkey_remote=?, shutdown_keyidx_local=?, channel_config_local=?, last_tx=?, last_sig=?, last_was_revoke=?, min_possible_feerate=?, max_possible_feerate=?, msatoshi_to_us_min=?, msatoshi_to_us_max=?, feerate_base=?, feerate_ppm=?, remote_upfront_shutdown_script=?, option_static_remotekey=?, option_anchor_outputs=?, shutdown_scriptpubkey_local=? WHERE id=?" msgstr "" -#: wallet/wallet.c:1698 +#: wallet/wallet.c:1503 msgid "UPDATE channels SET fundingkey_remote=?, revocation_basepoint_remote=?, payment_basepoint_remote=?, htlc_basepoint_remote=?, delayed_payment_basepoint_remote=?, per_commit_remote=?, old_per_commit_remote=?, channel_config_remote=?, future_per_commitment_point=? WHERE id=?" msgstr "" -#: wallet/wallet.c:1725 +#: wallet/wallet.c:1530 msgid "DELETE FROM channel_feerates WHERE channel_id=?" msgstr "" -#: wallet/wallet.c:1735 +#: wallet/wallet.c:1540 msgid "INSERT INTO channel_feerates VALUES(?, ?, ?)" msgstr "" -#: wallet/wallet.c:1752 +#: wallet/wallet.c:1557 msgid "UPDATE channels SET last_sent_commit=? WHERE id=?" msgstr "" -#: wallet/wallet.c:1766 +#: wallet/wallet.c:1571 msgid "SELECT id FROM peers WHERE node_id = ?" msgstr "" -#: wallet/wallet.c:1778 +#: wallet/wallet.c:1583 msgid "UPDATE peers SET address = ? WHERE id = ?" msgstr "" -#: wallet/wallet.c:1787 +#: wallet/wallet.c:1592 msgid "INSERT INTO peers (node_id, address) VALUES (?, ?);" msgstr "" -#: wallet/wallet.c:1805 +#: wallet/wallet.c:1610 msgid "INSERT INTO channels (peer_id, first_blocknum, id) VALUES (?, ?, ?);" msgstr "" -#: wallet/wallet.c:1831 +#: wallet/wallet.c:1636 msgid "DELETE FROM channel_htlcs WHERE channel_id=?" msgstr "" -#: wallet/wallet.c:1837 +#: wallet/wallet.c:1642 msgid "DELETE FROM htlc_sigs WHERE channelid=?" msgstr "" -#: wallet/wallet.c:1843 +#: wallet/wallet.c:1648 msgid "DELETE FROM channeltxs WHERE channel_id=?" msgstr "" -#: wallet/wallet.c:1849 +#: wallet/wallet.c:1654 msgid "DELETE FROM shachains WHERE id IN ( SELECT shachain_remote_id FROM channels WHERE channels.id=?)" msgstr "" -#: wallet/wallet.c:1859 +#: wallet/wallet.c:1664 msgid "UPDATE channels SET state=?, peer_id=?WHERE channels.id=?" msgstr "" -#: wallet/wallet.c:1873 +#: wallet/wallet.c:1678 msgid "SELECT * FROM channels WHERE peer_id = ?;" msgstr "" -#: wallet/wallet.c:1881 +#: wallet/wallet.c:1686 msgid "DELETE FROM peers WHERE id=?" msgstr "" -#: wallet/wallet.c:1892 +#: wallet/wallet.c:1697 msgid "UPDATE outputs SET confirmation_height = ? WHERE prev_out_tx = ?" msgstr "" -#: wallet/wallet.c:1995 +#: wallet/wallet.c:1800 msgid "INSERT INTO channel_htlcs ( channel_id, channel_htlc_id, direction, msatoshi, cltv_expiry, payment_hash, payment_key, hstate, shared_secret, routing_onion, received_time) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?);" msgstr "" -#: wallet/wallet.c:2048 +#: wallet/wallet.c:1853 msgid "INSERT INTO channel_htlcs ( channel_id, channel_htlc_id, direction, origin_htlc, msatoshi, cltv_expiry, payment_hash, payment_key, hstate, routing_onion, partid) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?);" msgstr "" -#: wallet/wallet.c:2108 +#: wallet/wallet.c:1913 msgid "UPDATE channel_htlcs SET hstate=?, payment_key=?, malformed_onion=?, failuremsg=?, localfailmsg=?, we_filled=? WHERE id=?" msgstr "" -#: wallet/wallet.c:2324 +#: wallet/wallet.c:2129 msgid "SELECT id, channel_htlc_id, msatoshi, cltv_expiry, hstate, payment_hash, payment_key, routing_onion, failuremsg, malformed_onion, origin_htlc, shared_secret, received_time, we_filled FROM channel_htlcs WHERE direction= ? AND channel_id= ? AND hstate != ?" msgstr "" -#: wallet/wallet.c:2371 +#: wallet/wallet.c:2176 msgid "SELECT id, channel_htlc_id, msatoshi, cltv_expiry, hstate, payment_hash, payment_key, routing_onion, failuremsg, malformed_onion, origin_htlc, shared_secret, received_time, partid, localfailmsg FROM channel_htlcs WHERE direction = ? AND channel_id = ? AND hstate != ?" msgstr "" -#: wallet/wallet.c:2501 +#: wallet/wallet.c:2306 msgid "SELECT channel_id, direction, cltv_expiry, channel_htlc_id, payment_hash FROM channel_htlcs WHERE channel_id = ?;" msgstr "" -#: wallet/wallet.c:2535 +#: wallet/wallet.c:2340 msgid "DELETE FROM channel_htlcs WHERE direction = ? AND origin_htlc = ? AND payment_hash = ? AND partid = ?;" msgstr "" -#: wallet/wallet.c:2588 +#: wallet/wallet.c:2393 msgid "SELECT status FROM payments WHERE payment_hash=? AND partid = ?;" msgstr "" -#: wallet/wallet.c:2606 +#: wallet/wallet.c:2411 msgid "INSERT INTO payments ( status, payment_hash, destination, msatoshi, timestamp, path_secrets, route_nodes, route_channels, msatoshi_sent, description, bolt11, total_msat, partid) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?);" msgstr "" -#: wallet/wallet.c:2689 +#: wallet/wallet.c:2494 msgid "DELETE FROM payments WHERE payment_hash = ? AND partid = ?" msgstr "" -#: wallet/wallet.c:2787 +#: wallet/wallet.c:2592 msgid "SELECT id, status, destination, msatoshi, payment_hash, timestamp, payment_preimage, path_secrets, route_nodes, route_channels, msatoshi_sent, description, bolt11, failonionreply, total_msat, partid FROM payments WHERE payment_hash = ? AND partid = ?" msgstr "" -#: wallet/wallet.c:2836 +#: wallet/wallet.c:2641 msgid "UPDATE payments SET status=? WHERE payment_hash=? AND partid=?" msgstr "" -#: wallet/wallet.c:2846 +#: wallet/wallet.c:2651 msgid "UPDATE payments SET payment_preimage=? WHERE payment_hash=? AND partid=?" msgstr "" -#: wallet/wallet.c:2856 +#: wallet/wallet.c:2661 msgid "UPDATE payments SET path_secrets = NULL , route_nodes = NULL , route_channels = NULL WHERE payment_hash = ? AND partid = ?;" msgstr "" -#: wallet/wallet.c:2888 +#: wallet/wallet.c:2693 msgid "SELECT failonionreply, faildestperm, failindex, failcode, failnode, failchannel, failupdate, faildetail, faildirection FROM payments WHERE payment_hash=? AND partid=?;" msgstr "" -#: wallet/wallet.c:2955 +#: wallet/wallet.c:2760 msgid "UPDATE payments SET failonionreply=? , faildestperm=? , failindex=? , failcode=? , failnode=? , failchannel=? , failupdate=? , faildetail=? , faildirection=? WHERE payment_hash=? AND partid=?;" msgstr "" -#: wallet/wallet.c:3014 +#: wallet/wallet.c:2819 msgid "SELECT id, status, destination, msatoshi, payment_hash, timestamp, payment_preimage, path_secrets, route_nodes, route_channels, msatoshi_sent, description, bolt11, failonionreply, total_msat, partid FROM payments WHERE payment_hash = ?;" msgstr "" -#: wallet/wallet.c:3035 +#: wallet/wallet.c:2840 msgid "SELECT id, status, destination, msatoshi, payment_hash, timestamp, payment_preimage, path_secrets, route_nodes, route_channels, msatoshi_sent, description, bolt11, failonionreply, total_msat, partid FROM payments ORDER BY id;" msgstr "" -#: wallet/wallet.c:3079 +#: wallet/wallet.c:2884 msgid "DELETE FROM htlc_sigs WHERE channelid = ?" msgstr "" -#: wallet/wallet.c:3086 +#: wallet/wallet.c:2891 msgid "INSERT INTO htlc_sigs (channelid, signature) VALUES (?, ?)" msgstr "" -#: wallet/wallet.c:3098 +#: wallet/wallet.c:2903 msgid "SELECT blobval FROM vars WHERE name='genesis_hash'" msgstr "" -#: wallet/wallet.c:3122 +#: wallet/wallet.c:2927 msgid "INSERT INTO vars (name, blobval) VALUES ('genesis_hash', ?);" msgstr "" -#: wallet/wallet.c:3140 +#: wallet/wallet.c:2945 msgid "SELECT txid, outnum FROM utxoset WHERE spendheight < ?" msgstr "" -#: wallet/wallet.c:3152 +#: wallet/wallet.c:2957 msgid "DELETE FROM utxoset WHERE spendheight < ?" msgstr "" -#: wallet/wallet.c:3160 wallet/wallet.c:3304 +#: wallet/wallet.c:2965 wallet/wallet.c:3111 msgid "INSERT INTO blocks (height, hash, prev_hash) VALUES (?, ?, ?);" msgstr "" -#: wallet/wallet.c:3179 +#: wallet/wallet.c:2984 msgid "DELETE FROM blocks WHERE hash = ?" msgstr "" -#: wallet/wallet.c:3185 +#: wallet/wallet.c:2990 msgid "SELECT * FROM blocks WHERE height >= ?;" msgstr "" -#: wallet/wallet.c:3194 +#: wallet/wallet.c:2999 msgid "DELETE FROM blocks WHERE height > ?" msgstr "" -#: wallet/wallet.c:3210 -msgid "UPDATE outputs SET spend_height = ? WHERE prev_out_tx = ? AND prev_out_index = ?" +#: wallet/wallet.c:3015 +msgid "UPDATE outputs SET spend_height = ?, status = ? WHERE prev_out_tx = ? AND prev_out_index = ?" msgstr "" -#: wallet/wallet.c:3226 +#: wallet/wallet.c:3033 msgid "UPDATE utxoset SET spendheight = ? WHERE txid = ? AND outnum = ?" msgstr "" -#: wallet/wallet.c:3245 +#: wallet/wallet.c:3052 msgid "SELECT blockheight, txindex FROM utxoset WHERE txid = ? AND outnum = ?" msgstr "" -#: wallet/wallet.c:3277 wallet/wallet.c:3315 +#: wallet/wallet.c:3084 wallet/wallet.c:3122 msgid "INSERT INTO utxoset ( txid, outnum, blockheight, spendheight, txindex, scriptpubkey, satoshis) VALUES(?, ?, ?, ?, ?, ?, ?);" msgstr "" -#: wallet/wallet.c:3341 +#: wallet/wallet.c:3148 msgid "SELECT height FROM blocks WHERE height = ?" msgstr "" -#: wallet/wallet.c:3354 +#: wallet/wallet.c:3161 msgid "SELECT txid, spendheight, scriptpubkey, satoshis FROM utxoset WHERE blockheight = ? AND txindex = ? AND outnum = ? AND spendheight IS NULL" msgstr "" -#: wallet/wallet.c:3396 wallet/wallet.c:3556 +#: wallet/wallet.c:3203 wallet/wallet.c:3363 msgid "SELECT blockheight FROM transactions WHERE id=?" msgstr "" -#: wallet/wallet.c:3406 +#: wallet/wallet.c:3213 msgid "INSERT INTO transactions ( id, blockheight, txindex, rawtx) VALUES (?, ?, ?, ?);" msgstr "" -#: wallet/wallet.c:3427 +#: wallet/wallet.c:3234 msgid "UPDATE transactions SET blockheight = ?, txindex = ? WHERE id = ?" msgstr "" -#: wallet/wallet.c:3444 +#: wallet/wallet.c:3251 msgid "INSERT INTO transaction_annotations (txid, idx, location, type, channel) VALUES (?, ?, ?, ?, ?) ON CONFLICT(txid,idx) DO NOTHING;" msgstr "" -#: wallet/wallet.c:3476 +#: wallet/wallet.c:3283 msgid "SELECT type, channel_id FROM transactions WHERE id=?" msgstr "" -#: wallet/wallet.c:3492 +#: wallet/wallet.c:3299 msgid "UPDATE transactions SET type = ?, channel_id = ? WHERE id = ?" msgstr "" -#: wallet/wallet.c:3511 +#: wallet/wallet.c:3318 msgid "SELECT type FROM transactions WHERE id=?" msgstr "" -#: wallet/wallet.c:3534 +#: wallet/wallet.c:3341 msgid "SELECT rawtx FROM transactions WHERE id=?" msgstr "" -#: wallet/wallet.c:3580 +#: wallet/wallet.c:3387 msgid "SELECT blockheight, txindex FROM transactions WHERE id=?" msgstr "" -#: wallet/wallet.c:3608 +#: wallet/wallet.c:3415 msgid "SELECT id FROM transactions WHERE blockheight=?" msgstr "" -#: wallet/wallet.c:3627 +#: wallet/wallet.c:3434 msgid "INSERT INTO channeltxs ( channel_id, type, transaction_id, input_num, blockheight) VALUES (?, ?, ?, ?, ?);" msgstr "" -#: wallet/wallet.c:3651 +#: wallet/wallet.c:3458 msgid "SELECT DISTINCT(channel_id) FROM channeltxs WHERE type = ?;" msgstr "" -#: wallet/wallet.c:3672 +#: wallet/wallet.c:3479 msgid "SELECT c.type, c.blockheight, t.rawtx, c.input_num, c.blockheight - t.blockheight + 1 AS depth, t.id as txid FROM channeltxs c JOIN transactions t ON t.id = c.transaction_id WHERE c.channel_id = ? ORDER BY c.id ASC;" msgstr "" -#: wallet/wallet.c:3717 +#: wallet/wallet.c:3524 msgid "UPDATE forwarded_payments SET in_msatoshi=?, out_msatoshi=?, state=?, resolved_time=?, failcode=? WHERE in_htlc_id=?" msgstr "" -#: wallet/wallet.c:3775 +#: wallet/wallet.c:3582 msgid "INSERT INTO forwarded_payments ( in_htlc_id, out_htlc_id, in_channel_scid, out_channel_scid, in_msatoshi, out_msatoshi, state, received_time, resolved_time, failcode) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?);" msgstr "" -#: wallet/wallet.c:3834 +#: wallet/wallet.c:3641 msgid "SELECT CAST(COALESCE(SUM(in_msatoshi - out_msatoshi), 0) AS BIGINT)FROM forwarded_payments WHERE state = ?;" msgstr "" -#: wallet/wallet.c:3858 +#: wallet/wallet.c:3665 msgid "SELECT f.state, in_msatoshi, out_msatoshi, hin.payment_hash as payment_hash, in_channel_scid, out_channel_scid, f.received_time, f.resolved_time, f.failcode FROM forwarded_payments f LEFT JOIN channel_htlcs hin ON (f.in_htlc_id = hin.id)" msgstr "" -#: wallet/wallet.c:4025 +#: wallet/wallet.c:3753 msgid "SELECT t.id, t.rawtx, t.blockheight, t.txindex, t.type as txtype, c2.short_channel_id as txchan, a.location, a.idx as ann_idx, a.type as annotation_type, c.short_channel_id FROM transactions t LEFT JOIN transaction_annotations a ON (a.txid = t.id) LEFT JOIN channels c ON (a.channel = c.id) LEFT JOIN channels c2 ON (t.channel_id = c2.id) ORDER BY t.blockheight, t.txindex ASC" msgstr "" -#: wallet/wallet.c:4119 +#: wallet/wallet.c:3847 msgid "INSERT INTO penalty_bases ( channel_id, commitnum, txid, outnum, amount) VALUES (?, ?, ?, ?, ?);" msgstr "" -#: wallet/wallet.c:4144 +#: wallet/wallet.c:3872 msgid "SELECT commitnum, txid, outnum, amount FROM penalty_bases WHERE channel_id = ?" msgstr "" -#: wallet/wallet.c:4168 +#: wallet/wallet.c:3896 msgid "DELETE FROM penalty_bases WHERE channel_id = ? AND commitnum = ?" msgstr "" @@ -1078,7 +1078,7 @@ msgstr "" msgid "not a valid SQL statement" msgstr "" -#: wallet/test/run-wallet.c:1334 +#: wallet/test/run-wallet.c:1345 msgid "INSERT INTO channels (id) VALUES (1);" msgstr "" -# SHA256STAMP:85251b488a498345744cf49002b812dd1fa9e88cf0c2abbf474cd51d81352397 +# SHA256STAMP:b285290a275969586a0f96adf2f00a52d79b17dd48192976c341be6beed1daeb From c2f5e8ce73efbf8c66c296d4e829eb482673ae5e Mon Sep 17 00:00:00 2001 From: Michael Schmoock Date: Thu, 3 Sep 2020 13:29:23 +0200 Subject: [PATCH 28/30] log: align level tags IO DEBUG INFO and UNUSUAL This aligns all log TAGs except **BROKEN** which would require too many spaces for all the other logs. Changelog-None --- lightningd/log.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lightningd/log.c b/lightningd/log.c index 139baa29d865..a5e57fd05f07 100644 --- a/lightningd/log.c +++ b/lightningd/log.c @@ -106,11 +106,11 @@ static const char *level_prefix(enum log_level level) switch (level) { case LOG_IO_OUT: case LOG_IO_IN: - return "IO"; + return "IO "; case LOG_DBG: - return "DEBUG"; + return "DEBUG "; case LOG_INFORM: - return "INFO"; + return "INFO "; case LOG_UNUSUAL: return "UNUSUAL"; case LOG_BROKEN: From 422607d1085ed77698e9708bb91e7cde7b3f8b70 Mon Sep 17 00:00:00 2001 From: Rusty Russell Date: Tue, 8 Sep 2020 14:50:13 +0930 Subject: [PATCH 29/30] lightning-cli: fix handling of complex objects with -H. I crashed it using `-H listconfigs` Signed-off-by: Rusty Russell Changelog-Fixed: cli: fixed crash with `listconfigs` in `-H` mode --- cli/lightning-cli.c | 2 +- cli/test/run-human-mode.c | 189 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 190 insertions(+), 1 deletion(-) create mode 100644 cli/test/run-human-mode.c diff --git a/cli/lightning-cli.c b/cli/lightning-cli.c index 692aafd7bd17..628684f37a4f 100644 --- a/cli/lightning-cli.c +++ b/cli/lightning-cli.c @@ -65,7 +65,7 @@ static size_t human_readable(const char *buffer, const jsmntok_t *t, char term) case JSMN_OBJECT: /* Elide single-field objects */ if (t->size == 1) - return human_readable(buffer, t + 2, '\n') + 3; + return human_readable(buffer, t + 2, '\n') + 2; n = 1; for (i = 0; i < t->size; i++) { n += human_readable(buffer, t + n, '='); diff --git a/cli/test/run-human-mode.c b/cli/test/run-human-mode.c new file mode 100644 index 000000000000..83b24b911851 --- /dev/null +++ b/cli/test/run-human-mode.c @@ -0,0 +1,189 @@ +#include "config.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +int test_main(int argc, char *argv[]); +ssize_t test_read(int fd, void *buf, size_t len); +int test_socket(int domain, int type, int protocol); +int test_connect(int sockfd, const struct sockaddr *addr, + socklen_t addrlen); +int test_getpid(void); +int test_printf(const char *format, ...); +int test_chdir(const char *path); + +#define main test_main +#define read test_read +#define socket test_socket +#define connect test_connect +#define getpid test_getpid +#define printf test_printf +#define chdir test_chdir + + #include "../lightning-cli.c" +#undef main + +/* AUTOGENERATED MOCKS START */ +/* Generated stub for amount_asset_extract_value */ +u8 *amount_asset_extract_value(const tal_t *ctx UNNEEDED, struct amount_asset *asset UNNEEDED) +{ fprintf(stderr, "amount_asset_extract_value called!\n"); abort(); } +/* Generated stub for amount_asset_is_main */ +bool amount_asset_is_main(struct amount_asset *asset UNNEEDED) +{ fprintf(stderr, "amount_asset_is_main called!\n"); abort(); } +/* Generated stub for amount_asset_to_sat */ +struct amount_sat amount_asset_to_sat(struct amount_asset *asset UNNEEDED) +{ fprintf(stderr, "amount_asset_to_sat called!\n"); abort(); } +/* Generated stub for amount_sat */ +struct amount_sat amount_sat(u64 satoshis UNNEEDED) +{ fprintf(stderr, "amount_sat called!\n"); abort(); } +/* Generated stub for amount_sat_add */ + bool amount_sat_add(struct amount_sat *val UNNEEDED, + struct amount_sat a UNNEEDED, + struct amount_sat b UNNEEDED) +{ fprintf(stderr, "amount_sat_add called!\n"); abort(); } +/* Generated stub for amount_sat_eq */ +bool amount_sat_eq(struct amount_sat a UNNEEDED, struct amount_sat b UNNEEDED) +{ fprintf(stderr, "amount_sat_eq called!\n"); abort(); } +/* Generated stub for amount_sat_greater_eq */ +bool amount_sat_greater_eq(struct amount_sat a UNNEEDED, struct amount_sat b UNNEEDED) +{ fprintf(stderr, "amount_sat_greater_eq called!\n"); abort(); } +/* Generated stub for amount_sat_sub */ + bool amount_sat_sub(struct amount_sat *val UNNEEDED, + struct amount_sat a UNNEEDED, + struct amount_sat b UNNEEDED) +{ fprintf(stderr, "amount_sat_sub called!\n"); abort(); } +/* Generated stub for amount_sat_to_asset */ +struct amount_asset amount_sat_to_asset(struct amount_sat *sat UNNEEDED, const u8 *asset UNNEEDED) +{ fprintf(stderr, "amount_sat_to_asset called!\n"); abort(); } +/* Generated stub for amount_tx_fee */ +struct amount_sat amount_tx_fee(u32 fee_per_kw UNNEEDED, size_t weight UNNEEDED) +{ fprintf(stderr, "amount_tx_fee called!\n"); abort(); } +/* Generated stub for fromwire_amount_msat */ +struct amount_msat fromwire_amount_msat(const u8 **cursor UNNEEDED, size_t *max UNNEEDED) +{ fprintf(stderr, "fromwire_amount_msat called!\n"); abort(); } +/* Generated stub for fromwire_amount_sat */ +struct amount_sat fromwire_amount_sat(const u8 **cursor UNNEEDED, size_t *max UNNEEDED) +{ fprintf(stderr, "fromwire_amount_sat called!\n"); abort(); } +/* Generated stub for fromwire_bigsize */ +bigsize_t fromwire_bigsize(const u8 **cursor UNNEEDED, size_t *max UNNEEDED) +{ fprintf(stderr, "fromwire_bigsize called!\n"); abort(); } +/* Generated stub for fromwire_channel_id */ +void fromwire_channel_id(const u8 **cursor UNNEEDED, size_t *max UNNEEDED, + struct channel_id *channel_id UNNEEDED) +{ fprintf(stderr, "fromwire_channel_id called!\n"); abort(); } +/* Generated stub for fromwire_node_id */ +void fromwire_node_id(const u8 **cursor UNNEEDED, size_t *max UNNEEDED, struct node_id *id UNNEEDED) +{ fprintf(stderr, "fromwire_node_id called!\n"); abort(); } +/* Generated stub for json_add_member */ +void json_add_member(struct json_stream *js UNNEEDED, + const char *fieldname UNNEEDED, + bool quote UNNEEDED, + const char *fmt UNNEEDED, ...) +{ fprintf(stderr, "json_add_member called!\n"); abort(); } +/* Generated stub for json_array_end */ +void json_array_end(struct json_stream *js UNNEEDED) +{ fprintf(stderr, "json_array_end called!\n"); abort(); } +/* Generated stub for json_array_start */ +void json_array_start(struct json_stream *js UNNEEDED, const char *fieldname UNNEEDED) +{ fprintf(stderr, "json_array_start called!\n"); abort(); } +/* Generated stub for json_member_direct */ +char *json_member_direct(struct json_stream *js UNNEEDED, + const char *fieldname UNNEEDED, size_t extra UNNEEDED) +{ fprintf(stderr, "json_member_direct called!\n"); abort(); } +/* Generated stub for json_object_end */ +void json_object_end(struct json_stream *js UNNEEDED) +{ fprintf(stderr, "json_object_end called!\n"); abort(); } +/* Generated stub for json_object_start */ +void json_object_start(struct json_stream *ks UNNEEDED, const char *fieldname UNNEEDED) +{ fprintf(stderr, "json_object_start called!\n"); abort(); } +/* Generated stub for towire_amount_msat */ +void towire_amount_msat(u8 **pptr UNNEEDED, const struct amount_msat msat UNNEEDED) +{ fprintf(stderr, "towire_amount_msat called!\n"); abort(); } +/* Generated stub for towire_amount_sat */ +void towire_amount_sat(u8 **pptr UNNEEDED, const struct amount_sat sat UNNEEDED) +{ fprintf(stderr, "towire_amount_sat called!\n"); abort(); } +/* Generated stub for towire_bigsize */ +void towire_bigsize(u8 **pptr UNNEEDED, const bigsize_t val UNNEEDED) +{ fprintf(stderr, "towire_bigsize called!\n"); abort(); } +/* Generated stub for towire_channel_id */ +void towire_channel_id(u8 **pptr UNNEEDED, const struct channel_id *channel_id UNNEEDED) +{ fprintf(stderr, "towire_channel_id called!\n"); abort(); } +/* Generated stub for towire_node_id */ +void towire_node_id(u8 **pptr UNNEEDED, const struct node_id *id UNNEEDED) +{ fprintf(stderr, "towire_node_id called!\n"); abort(); } +/* Generated stub for version_and_exit */ +char *version_and_exit(const void *unused UNNEEDED) +{ fprintf(stderr, "version_and_exit called!\n"); abort(); } +/* AUTOGENERATED MOCKS END */ + +int test_socket(int domain UNUSED, int type UNUSED, int protocol UNUSED) +{ + /* We give a real fd, as it writes to it */ + return open("/dev/null", O_WRONLY); +} + +int test_connect(int sockfd UNUSED, const struct sockaddr *addr UNUSED, + socklen_t addrlen UNUSED) +{ + return 0; +} + +int test_getpid(void) +{ + return 9999; +} + +int test_printf(const char *fmt UNUSED, ...) +{ + return 0; +} + +int test_chdir(const char *path) +{ + return 0; +} + +static const char *response = + "{ \"jsonrpc\":\"2.0\"," + "\"id\":\"lightning-cli-9999\"," + "\"result\":" + "{\"# version\":\"v0.9.0-240-gd7ff74e-modded\",\"lightning-dir\":\"/home/rusty/.lightning\",\"network\":\"testnet\",\"allow-deprecated-apis\":false,\"rpc-file\":\"lightning-rpc\",\"plugins\":[{\"path\":\"/home/rusty/.lightning/plugins/summary/summary.py\",\"name\":\"summary.py\",\"options\":{\"summary-currency\":\"USD\",\"summary-currency-prefix\":\"USD $\",\"summary-availability-interval\":\"300\",\"summary-availability-window\":\"72\"}}],\"important-plugins\":[{\"path\":\"/home/rusty/lightning/lightningd/../plugins/autoclean\",\"name\":\"autoclean\",\"options\":{\"autocleaninvoice-cycle\":null,\"autocleaninvoice-expired-by\":null}},{\"path\":\"/home/rusty/lightning/lightningd/../plugins/bcli\",\"name\":\"bcli\",\"options\":{\"bitcoin-datadir\":null,\"bitcoin-cli\":null,\"bitcoin-rpcuser\":null,\"bitcoin-rpcpassword\":null,\"bitcoin-rpcconnect\":null,\"bitcoin-rpcport\":null,\"bitcoin-retry-timeout\":null,\"commit-fee\":null,\"dev-max-fee-multiplier\":null}},{\"path\":\"/home/rusty/lightning/lightningd/../plugins/fundchannel\",\"name\":\"fundchannel\"},{\"path\":\"/home/rusty/lightning/lightningd/../plugins/keysend\",\"name\":\"keysend\"},{\"path\":\"/home/rusty/lightning/lightningd/../plugins/pay\",\"name\":\"pay\",\"options\":{\"disable-mpp\":false}}],\"disable-plugin\":[],\"always-use-proxy\":false,\"daemon\":\"false\",\"wallet\":\"sqlite3:///home/rusty/.lightning/testnet/lightningd.sqlite3\",\"wumbo\":false,\"wumbo\":false,\"rgb\":\"031a34\",\"alias\":\"SLICKERMASTER-40-gd7ff74e-modded\",\"pid-file\":\"/home/rusty/.lightning/lightningd-testnet.pid\",\"ignore-fee-limits\":true,\"watchtime-blocks\":6,\"max-locktime-blocks\":2016,\"funding-confirms\":1,\"commit-fee-min\":0,\"commit-fee-max\":0,\"cltv-delta\":6,\"cltv-final\":10,\"commit-time\":10,\"fee-base\":1,\"rescan\":30,\"fee-per-satoshi\":10,\"max-concurrent-htlcs\":483,\"min-capacity-sat\":10000,\"bind-addr\":\":9736\",\"announce-addr\":\"128.199.202.168:9736\",\"announce-addr\":\"[2400:6180:0:d0::5cd2:a001]:9736\",\"offline\":\"false\",\"autolisten\":true,\"disable-dns\":\"false\",\"enable-autotor-v2-mode\":\"false\",\"encrypted-hsm\":false,\"rpc-file-mode\":\"0600\",\"log-level\":\"DEBUG\",\"log-prefix\":\"lightningd\",\"log-file\":\"/home/rusty/logs/lightningd-testnet.log\",\"dev-no-reconnect\":\"false\",\"dev-allow-localhost\":\"false\",\"dev-bitcoind-poll\":30,\"dev-fast-gossip\":\"false\",\"dev-fast-gossip-prune\":\"false\",\"dev-gossip-time\":0,\"dev-max-funding-unconfirmed-blocks\":2016,\"dev-no-htlc-timeout\":\"false\",\"dev-fail-process-onionpacket\":\"false\",\"dev-no-version-checks\":\"false\",\"dev-builtin-plugins-unimportant\":\"false\"}" + "}\n\n"; + +static size_t response_off, max_read_return; + +ssize_t test_read(int fd UNUSED, void *buf, size_t len) +{ + if (len > max_read_return) + len = max_read_return; + if (len > strlen(response + response_off)) + len = strlen(response + response_off); + + memcpy(buf, response + response_off, len); + response_off += len; + return len; +} + +int main(int argc UNUSED, char *argv[]) +{ + setup_locale(); + + char *fake_argv[] = { argv[0], "--lightning-dir=/tmp/", "-H", "listconfigs", NULL }; + + response_off = 0; + max_read_return = -1; + assert(test_main(4, fake_argv) == 0); + assert(!taken_any()); + take_cleanup(); + return 0; +} + From d7a60a98f1bbeea26d71f0c5b894ac42eedb2de6 Mon Sep 17 00:00:00 2001 From: Rusty Russell Date: Tue, 8 Sep 2020 14:50:15 +0930 Subject: [PATCH 30/30] options: handle wumbo and large-channels aliases properly. Too trivial a fix to really list in Changelog, but I noticed that we specified "wumbo" twice. We should really just use the proper name in listconfigs. Signed-off-by: Rusty Russell --- lightningd/options.c | 5 ++++- tests/test_misc.py | 4 ++++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/lightningd/options.c b/lightningd/options.c index 1b8f5a113a69..f795ac845bae 100644 --- a/lightningd/options.c +++ b/lightningd/options.c @@ -1241,7 +1241,7 @@ static void add_config(struct lightningd *ld, } else if (opt->cb == (void *)opt_set_hsm_password) { json_add_bool(response, "encrypted-hsm", ld->encrypted_hsm); } else if (opt->cb == (void *)opt_set_wumbo) { - json_add_bool(response, "wumbo", + json_add_bool(response, name0, feature_offered(ld->our_features ->bits[INIT_FEATURE], OPT_LARGE_CHANNELS)); @@ -1387,6 +1387,9 @@ static struct command_result *json_listconfigs(struct command *cmd, response = json_stream_success(cmd); add_config(cmd->ld, response, &opt_table[i], name+1, len-1); + /* If we have more than one long name, first + * is preferred */ + break; } } diff --git a/tests/test_misc.py b/tests/test_misc.py index 491dd6fead07..e9d7f51bd9e8 100644 --- a/tests/test_misc.py +++ b/tests/test_misc.py @@ -807,6 +807,10 @@ def test_listconfigs(node_factory, bitcoind, chainparams): assert configs['ignore-fee-limits'] is False assert configs['log-prefix'] == 'lightning1-xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx...' + # These are aliases, but we don't print the (unofficial!) wumbo. + assert 'wumbo' not in configs + assert configs['large-channels'] is False + # Test one at a time. for c in configs.keys(): if c.startswith('#') or c.startswith('plugins') or c == 'important-plugins':