Merge with upstream

.gitignore

@@ -10,6 +10,7 @@
 *.rej
 .cppcheck-suppress
 TAGS
+tags
 ccan/tools/configurator/configurator
 ccan/ccan/cdump/tools/cdump-enumstr
 gen_*
@@ -21,32 +22,11 @@ __pycache__
 config.vars
 
 # Ignore some generated binaries
-lightningd/test/run-channel
-lightningd/test/run-cryptomsg
-lightningd/test/run-commit_tx
-lightningd/test/run-find_my_path
-lightningd/test/run-funding_tx
-lightningd/test/run-param
-lightningd/test/run-key_derive
-common/test/run-ip_port_parsing
-wire/test/run-peer-wire
-bitcoin/test/run-tx-encode
-channeld/test/run-full_channel
-common/test/run-bolt11
-common/test/run-derive_basepoints
-common/test/run-features
-common/test/run-json
-common/test/run-sphinx
-connectd/test/run-initiator-success
-connectd/test/run-responder-success
-daemon/test/run-maxfee
+*/test/run-*
+!*/test/run-*.c
 external/libbacktrace-build/
 external/libbacktrace.a
 external/libbacktrace.la
-gossipd/test/run-find_route-specific
-gossipd/test/run-initiator-success
-gossipd/test/run-responder-success
-onchaind/test/run-grind_feerate
 test/test_protocol
 test/test_sphinx
 tests/.pytest.restart

CHANGELOG.md

@@ -9,16 +9,21 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.
 ### Added
 
 - JSON API: `listpeers` has new field `scratch_txid`: the latest tx in channel.
+- JSON API: `listchannels` has two new fields: `message_flags` and `channel_flags`. This replaces `flags`.
 - Bitcoind: more parallelism in requests, for very slow nodes.
 - Testing: fixed logging, cleaner interception of bitcoind, minor fixes.
 
 ### Changed
 
+- Protocol: `channel_update` sent to disable channel only if we reject an HTLC.
+
 ### Deprecated
 
 Note: You should always set `allow-deprecated-apis=false` to test for
 changes.
 
+- JSON RPC: `listchannels`' `flags` field. This has been split into two fields, see Added.
+
 ### Removed
 
 ### Fixed

Makefile

@@ -9,7 +9,7 @@ CCANDIR := ccan
 
 # Where we keep the BOLT RFCs
 BOLTDIR := ../lightning-rfc/
-BOLTVERSION := fd9da9b95eb5d585252d7e749212151502e0cc17
+BOLTVERSION := 0891374d47ddffa64c5a2e6ad151247e3d6b7a59
 
 -include config.vars
 
@@ -243,12 +243,12 @@ check-makefile:
 
 # Any mention of BOLT# must be followed by an exact quote, modulo whitespace.
 bolt-check/%: % bolt-precheck tools/check-bolt
-	@[ ! -d .tmp.lightningrfc ] || tools/check-bolt .tmp.lightningrfc $<
+	@if [ -d .tmp.lightningrfc ]; then tools/check-bolt .tmp.lightningrfc $<; else Not checking BOLTs: BOLTDIR $(BOLTDIR) does not exist >&2; fi
 
 LOCAL_BOLTDIR=.tmp.lightningrfc
 
 bolt-precheck:
-	@rm -rf $(LOCAL_BOLTDIR); if [ ! -d $(BOLTDIR) ]; then echo Not checking BOLTs: BOLTDIR $(BOLTDIR) does not exist >&2; exit 0; fi; set -e; if [ -n "$(BOLTVERSION)" ]; then git clone -q $(BOLTDIR) $(LOCAL_BOLTDIR) && cd $(LOCAL_BOLTDIR) && git checkout -q $(BOLTVERSION); else cp -a $(BOLTDIR) $(LOCAL_BOLTDIR); fi
+	@[ -d $(BOLTDIR) ] || exit 0; set -e; if [ -z "$(BOLTVERSION)" ]; then rm -rf $(LOCAL_BOLTDIR); ln -sf $(BOLTDIR) $(LOCAL_BOLTDIR); exit 0; fi; [ "$$(git -C $(LOCAL_BOLTDIR) rev-list --max-count=1 HEAD 2>/dev/null)" != "$(BOLTVERSION)" ] || exit 0; rm -rf $(LOCAL_BOLTDIR) && git clone -q $(BOLTDIR) $(LOCAL_BOLTDIR) && cd $(LOCAL_BOLTDIR) && git checkout -q $(BOLTVERSION)
 
 check-source-bolt: $(ALL_TEST_PROGRAMS:%=bolt-check/%.c)
 

channeld/commit_tx.c

@@ -200,7 +200,7 @@ struct bitcoin_tx *commit_tx(const tal_t *ctx,
 	 *
 	 * 5. If the `to_local` amount is greater or equal to
 	 *    `dust_limit_satoshis`, add a [`to_local`
-	 *    output](#to-local-output).
+	 *    output](#to_local-output).
 	 */
 	if (self_pay_msat / 1000 >= dust_limit_satoshis) {
 		u8 *wscript = to_self_wscript(tmpctx, to_self_delay,keyset);
@@ -218,7 +218,7 @@ struct bitcoin_tx *commit_tx(const tal_t *ctx,
 	 *
 	 * 6. If the `to_remote` amount is greater or equal to
 	 *    `dust_limit_satoshis`, add a [`to_remote`
-	 *    output](#to-remote-output).
+	 *    output](#to_remote-output).
 	 */
 	if (other_pay_msat / 1000 >= dust_limit_satoshis) {
 		/* BOLT #3:

common/bolt11.c

@@ -498,7 +498,8 @@ struct bolt11 *bolt11_decode(const tal_t *ctx, const char *str,
 
         /* BOLT #11:
          *
-         * A reader MUST fail if it does not understand the `prefix`.
+         * A reader:
+         *   - MUST fail if it does not understand the `prefix`
          */
         if (!strstarts(prefix, "ln"))
                 return decode_fail(b11, fail,
@@ -510,14 +511,13 @@ struct bolt11 *bolt11_decode(const tal_t *ctx, const char *str,
 
         /* BOLT #11:
          *
-         * A reader SHOULD fail if `amount` contains a non-digit or
-         * is followed by anything except a `multiplier` in the table
-         * above. */
+         *   - If the `amount` is empty:
+         * */
         amountstr = tal_strdup(tmpctx, hrp + strlen(prefix));
         if (streq(amountstr, "")) {
                 /* BOLT #11:
                  *
-                 * A reader SHOULD indicate if amount is unspecified
+	         * - SHOULD indicate if amount is unspecified
                  */
                 b11->msatoshi = NULL;
         } else {
@@ -537,10 +537,9 @@ struct bolt11 *bolt11_decode(const tal_t *ctx, const char *str,
 
                 /* BOLT #11:
                  *
-                 * A reader SHOULD fail if `amount` contains a non-digit or
-                 * is followed by anything except a `multiplier` in the table
-                 * above.
-                 */
+                 * MUST fail if `amount` contains a non-digit or is followed by
+                 * anything except a `multiplier` in the table above
+                 **/
                 amount = strtoull(amountstr, &end, 10);
                 if (amount == ULLONG_MAX && errno == ERANGE)
                         return decode_fail(b11, fail,
@@ -549,7 +548,12 @@ struct bolt11 *bolt11_decode(const tal_t *ctx, const char *str,
                         return decode_fail(b11, fail,
                                            "Invalid amount postfix '%s'", end);
 
-                /* Convert to millisatoshis. */
+                /* BOLT #11:
+                 *
+                 * - If the `multiplier` is present:
+                 *   - MUST multiply `amount` by the `multiplier`
+                 *   value to derive the amount required for payment
+                 **/
                 b11->msatoshi = tal(b11, u64);
                 *b11->msatoshi = amount * m10 / 10;
         }
@@ -873,9 +877,15 @@ char *bolt11_encode_(const tal_t *ctx,
 
         /* BOLT #11:
          *
-         * A writer MUST encode `amount` as a positive decimal integer
-         * with no leading zeroes and SHOULD use the shortest representation
-	 * possible.
+         * A writer:
+         *   - MUST encode `prefix` using the currency it requires
+         *   for successful payment
+         *   - If it requires a specific minimum amount for successful payment:
+	 *     - MUST include that `amount`
+	 *     - MUST encode `amount` as a positive decimal integer
+         *     with no leading zeroes
+	 *     - SHOULD use the shortest representation possible by
+         *     using the largest multiplier or omitting the multiplier
          */
         if (b11->msatoshi) {
                 char postfix;

common/initial_commit_tx.c

@@ -157,7 +157,7 @@ struct bitcoin_tx *initial_commit_tx(const tal_t *ctx,
 	 *
 	 * 5. If the `to_local` amount is greater or equal to
 	 *    `dust_limit_satoshis`, add a [`to_local`
-	 *    output](#to-local-output).
+	 *    output](#to_local-output).
 	 */
 	if (self_pay_msat / 1000 >= dust_limit_satoshis) {
 		u8 *wscript = to_self_wscript(tmpctx, to_self_delay,keyset);
@@ -170,7 +170,7 @@ struct bitcoin_tx *initial_commit_tx(const tal_t *ctx,
 	 *
 	 * 6. If the `to_remote` amount is greater or equal to
 	 *    `dust_limit_satoshis`, add a [`to_remote`
-	 *    output](#to-remote-output).
+	 *    output](#to_remote-output).
 	 */
 	if (other_pay_msat / 1000 >= dust_limit_satoshis) {
 		/* BOLT #3:

common/json.c

@@ -188,23 +188,34 @@ jsmntok_t *json_parse_input(const char *input, int len, bool *valid)
 	int ret;
 
 	toks = tal_arr(input, jsmntok_t, 10);
+	toks[0].type = JSMN_UNDEFINED;
 
-again:
 	jsmn_init(&parser);
+again:
 	ret = jsmn_parse(&parser, input, len, toks, tal_count(toks) - 1);
 
 	switch (ret) {
 	case JSMN_ERROR_INVAL:
 		*valid = false;
 		return tal_free(toks);
-	case JSMN_ERROR_PART:
-		*valid = true;
-		return tal_free(toks);
 	case JSMN_ERROR_NOMEM:
 		tal_resize(&toks, tal_count(toks) * 2);
 		goto again;
 	}
 
+	/* Check whether we read at least one full root element, i.e., root
+	 * element has its end set. */
+	if (toks[0].type == JSMN_UNDEFINED || toks[0].end == -1) {
+		*valid = true;
+		return tal_free(toks);
+	}
+
+	/* If we read a partial element at the end of the stream we'll get a
+	 * ret=JSMN_ERROR_PART, but due to the previous check we know we read at
+	 * least one full element, so count tokens that are part of this root
+	 * element. */
+	ret = json_next(toks) - toks;
+
 	/* Cut to length and return. */
 	*valid = true;
 	tal_resize(&toks, ret + 1);

common/test/run-json.c

@@ -145,6 +145,37 @@ static void test_json_partial(void)
 	tal_free(ctx);
 }
 
+/* Test that we can segment and parse a stream of json objects correctly. */
+static void test_json_stream(void)
+{
+	bool valid;
+	char *input, *talstr;
+	jsmntok_t *toks;
+
+	/* Multiple full messages in a single buffer (happens when buffer
+	 * boundary coincides with message boundary, or read returned after
+	 * timeout. */
+	input = "{\"x\":\"x\"}{\"y\":\"y\"}";
+	talstr = tal_strndup(NULL, input, strlen(input));
+	toks = json_parse_input(talstr, strlen(talstr), &valid);
+	assert(toks);
+	assert(tal_count(toks) == 4);
+	assert(toks[0].start == 0 && toks[0].end == 9);
+	assert(valid);
+	tal_free(talstr);
+
+	/* Multiple messages, and the last one is partial, far more likely than
+	 * accidentally getting the boundaries to match. */
+	input = "{\"x\":\"x\"}{\"y\":\"y\"}{\"z\":\"z";
+	talstr = tal_strndup(NULL, input, strlen(input));
+	toks = json_parse_input(talstr, strlen(talstr), &valid);
+	assert(toks);
+	assert(tal_count(toks) == 4);
+	assert(toks[0].start == 0 && toks[0].end == 9);
+	assert(valid);
+	tal_free(talstr);
+}
+
 int main(void)
 {
 	setup_locale();
@@ -153,6 +184,7 @@ int main(void)
 	test_json_filter();
 	test_json_escape();
 	test_json_partial();
+	test_json_stream();
 	assert(!taken_any());
 	take_cleanup();
 }

devtools/.gitignore

@@ -2,3 +2,4 @@ bolt11-cli
 decodemsg
 onion
 dump-gossipstore
+gossipwith

devtools/Makefile

@@ -1,6 +1,7 @@
 DEVTOOLS_SRC := devtools/gen_print_wire.c devtools/gen_print_onion_wire.c devtools/print_wire.c
 DEVTOOLS_OBJS := $(DEVTOOLS_SRC:.c=.o)
-DEVTOOLS_TOOL_SRC := devtools/bolt11-cli.c devtools/decodemsg.c devtools/onion.c devtools/dump-gossipstore.c
+DEVTOOLS := devtools/bolt11-cli devtools/decodemsg devtools/onion devtools/dump-gossipstore devtools/gossipwith
+DEVTOOLS_TOOL_SRC := $(DEVTOOLS:=.c)
 DEVTOOLS_TOOL_OBJS := $(DEVTOOLS_TOOL_SRC:.c=.o)
 
 DEVTOOLS_COMMON_OBJS :=				\
@@ -15,7 +16,7 @@ DEVTOOLS_COMMON_OBJS :=				\
 	common/version.o			\
 	common/wireaddr.o
 
-devtools-all: devtools/bolt11-cli devtools/decodemsg devtools/onion devtools/dump-gossipstore
+devtools-all: $(DEVTOOLS)
 
 devtools/gen_print_wire.h: $(WIRE_GEN) wire/gen_peer_wire_csv
 	$(WIRE_GEN) --bolt --printwire --header $@ wire_type < wire/gen_peer_wire_csv > $@
@@ -40,12 +41,16 @@ devtools/onion.c: ccan/config.h
 
 devtools/onion: $(DEVTOOLS_OBJS) $(DEVTOOLS_COMMON_OBJS) $(JSMN_OBJS) $(CCAN_OBJS) $(BITCOIN_OBJS) wire/fromwire.o wire/towire.o devtools/onion.o common/sphinx.o
 
+devtools/gossipwith: $(DEVTOOLS_OBJS) $(DEVTOOLS_COMMON_OBJS) $(JSMN_OBJS) $(CCAN_OBJS) $(BITCOIN_OBJS) wire/fromwire.o wire/towire.o wire/gen_peer_wire.o devtools/gossipwith.o common/cryptomsg.o common/cryptomsg.o common/crypto_sync.o
+
 $(DEVTOOLS_OBJS) $(DEVTOOLS_TOOL_OBJS): wire/wire.h devtools/gen_print_wire.h devtools/gen_print_onion_wire.h
 devtools/gen_print_wire.o: devtools/gen_print_wire.h wire/gen_peer_wire.h devtools/print_wire.h
 devtools/gen_print_onion_wire.o: devtools/gen_print_onion_wire.h devtools/print_wire.h
 
+devtools/bolt11-cli: $(DEVTOOLS_OBJS) $(DEVTOOLS_COMMON_OBJS) $(JSMN_OBJS) $(CCAN_OBJS) $(BITCOIN_OBJS) wire/fromwire.o wire/towire.o devtools/bolt11-cli.o
+
 # Make sure these depend on everything.
-ALL_PROGRAMS += devtools/bolt11-cli devtools/decodemsg devtools/onion devtools/dump-gossipstore
+ALL_PROGRAMS += $(DEVTOOLS)
 ALL_OBJS += $(DEVTOOLS_OBJS) $(DEVTOOLS_TOOL_OBJS)
 
 check-source: $(DEVTOOLS_SRC:%=check-src-include-order/%) $(DEVTOOLS_TOOLS_SRC:%=check-src-include-order/%)