Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Update routing graph with rate-limited gossip #5239

Merged
merged 4 commits into from
Jul 6, 2022
Merged

Update routing graph with rate-limited gossip #5239

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
ee908a6
gossip_store: add flag for spam gossip, update to v10
endothermicdev May 3, 2022
a1eae04
gossipd: make use of new ratelimit bit in gossip_store length mask
endothermicdev May 4, 2022
ca93240
pytest: test_gossip_ratelimit checks routing graph and squelch
endothermicdev May 4, 2022
d7ec151
gossipd: store and index most recent and last non-rate-limited gossip
endothermicdev May 6, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 5 additions & 1 deletion common/gossip_store.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -53,14 +53,15 @@ u8 *gossip_store_next(const tal_t *ctx,
int *gossip_store_fd,
u32 timestamp_min, u32 timestamp_max,
bool push_only,
bool with_spam,
size_t *off, size_t *end)
{
u8 *msg = NULL;

while (!msg) {
struct gossip_hdr hdr;
u32 msglen, checksum, timestamp;
bool push;
bool push, ratelimited;
int type, r;

r = pread(*gossip_store_fd, &hdr, sizeof(hdr), *off);
Expand All @@ -69,6 +70,7 @@ u8 *gossip_store_next(const tal_t *ctx,

msglen = be32_to_cpu(hdr.len);
push = (msglen & GOSSIP_STORE_LEN_PUSH_BIT);
ratelimited = (msglen & GOSSIP_STORE_LEN_RATELIMIT_BIT);
msglen &= GOSSIP_STORE_LEN_MASK;

/* Skip any deleted entries. */
Expand Down Expand Up @@ -115,6 +117,8 @@ u8 *gossip_store_next(const tal_t *ctx,
msg = tal_free(msg);
} else if (!push && push_only) {
msg = tal_free(msg);
} else if (!with_spam && ratelimited) {
msg = tal_free(msg);
}
}

Expand Down
11 changes: 9 additions & 2 deletions common/gossip_store.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@ struct gossip_rcvd_filter;
/**
* gossip_store -- On-disk storage related information
*/
#define GOSSIP_STORE_VERSION 9
#define GOSSIP_STORE_VERSION 10

/**
* Bit of length we use to mark a deleted record.
Expand All @@ -23,9 +23,15 @@ struct gossip_rcvd_filter;
*/
#define GOSSIP_STORE_LEN_PUSH_BIT 0x40000000U

/**
* Bit of length used to define a rate-limited record (do not rebroadcast)
*/
#define GOSSIP_STORE_LEN_RATELIMIT_BIT 0x20000000U

/* Mask for extracting just the length part of len field */
#define GOSSIP_STORE_LEN_MASK \
(~(GOSSIP_STORE_LEN_PUSH_BIT | GOSSIP_STORE_LEN_DELETED_BIT))
(~(GOSSIP_STORE_LEN_PUSH_BIT | GOSSIP_STORE_LEN_DELETED_BIT | \
GOSSIP_STORE_LEN_RATELIMIT_BIT))

/**
* gossip_hdr -- On-disk format header.
Expand All @@ -47,6 +53,7 @@ u8 *gossip_store_next(const tal_t *ctx,
int *gossip_store_fd,
u32 timestamp_min, u32 timestamp_max,
bool push_only,
bool with_spam,
size_t *off, size_t *end);

/**
Expand Down
2 changes: 1 addition & 1 deletion common/test/run-gossmap_local.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,7 @@ void towire_channel_id(u8 **pptr UNNEEDED, const struct channel_id *channel_id U
* setup_gossip_store_test via od -v -Anone -tx1 < /tmp/ltests-kaf30pn0/test_gossip_store_compact_noappend_1/lightning-2/regtest/gossip_store
*/
static u8 canned_map[] = {
0x09, 0x80, 0x00, 0x01, 0xbc, 0x09, 0x8b, 0x67, 0xe6, 0x00, 0x00, 0x00, 0x00, 0x10, 0x08, 0x00
0x0a, 0x80, 0x00, 0x01, 0xbc, 0x09, 0x8b, 0x67, 0xe6, 0x00, 0x00, 0x00, 0x00, 0x10, 0x08, 0x00
, 0x00, 0x00, 0x00, 0x00, 0x0f, 0x42, 0x40, 0x01, 0xb0, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00
, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00
, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00
Expand Down
2 changes: 2 additions & 0 deletions connectd/multiplex.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -375,6 +375,7 @@ static u8 *maybe_from_gossip_store(const tal_t *ctx, struct peer *peer)
return gossip_store_next(ctx, &peer->daemon->gossip_store_fd,
0, 0xFFFFFFFF,
true,
false,
&peer->gs.off,
&peer->daemon->gossip_store_end);
}
Expand All @@ -391,6 +392,7 @@ static u8 *maybe_from_gossip_store(const tal_t *ctx, struct peer *peer)
peer->gs.timestamp_min,
peer->gs.timestamp_max,
false,
false,
&peer->gs.off,
&peer->daemon->gossip_store_end);
/* Don't send back gossip they sent to us! */
Expand Down
12 changes: 7 additions & 5 deletions contrib/pyln-client/pyln/client/gossmap.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,11 +9,13 @@
import struct

# These duplicate constants in lightning/common/gossip_store.h
GOSSIP_STORE_VERSION = 9
GOSSIP_STORE_VERSIONS = [0x09, 0x0a]
GOSSIP_STORE_LEN_DELETED_BIT = 0x80000000
GOSSIP_STORE_LEN_PUSH_BIT = 0x40000000
GOSSIP_STORE_LEN_RATELIMIT_BIT = 0x20000000
GOSSIP_STORE_LEN_MASK = (~(GOSSIP_STORE_LEN_PUSH_BIT
| GOSSIP_STORE_LEN_DELETED_BIT))
| GOSSIP_STORE_LEN_DELETED_BIT
| GOSSIP_STORE_LEN_RATELIMIT_BIT))

# These duplicate constants in lightning/gossipd/gossip_store_wiregen.h
WIRE_GOSSIP_STORE_PRIVATE_CHANNEL = 4104
Expand Down Expand Up @@ -171,9 +173,9 @@ def __init__(self, store_filename: str = "gossip_store"):
self.nodes: Dict[GossmapNodeId, GossmapNode] = {}
self.channels: Dict[ShortChannelId, GossmapChannel] = {}
self._last_scid: Optional[str] = None
version = self.store_file.read(1)
if version[0] != GOSSIP_STORE_VERSION:
raise ValueError("Invalid gossip store version {}".format(int(version)))
version = self.store_file.read(1)[0]
if version not in GOSSIP_STORE_VERSIONS:
raise ValueError("Invalid gossip store version {}".format(version))
self.bytes_read = 1
self.refresh()

Expand Down
Binary file modified contrib/pyln-client/tests/data/gossip_store-part1.xz
View file
Open in desktop
Binary file not shown.
Binary file modified contrib/pyln-client/tests/data/gossip_store.simple.xz
View file
Open in desktop
Binary file not shown.
8 changes: 5 additions & 3 deletions devtools/dump-gossipstore.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -52,10 +52,11 @@ int main(int argc, char *argv[])
struct short_channel_id scid;
u32 msglen = be32_to_cpu(hdr.len);
u8 *msg, *inner;
bool deleted, push;
bool deleted, push, ratelimit;

deleted = (msglen & GOSSIP_STORE_LEN_DELETED_BIT);
push = (msglen & GOSSIP_STORE_LEN_PUSH_BIT);
ratelimit = (msglen & GOSSIP_STORE_LEN_RATELIMIT_BIT);

msglen &= GOSSIP_STORE_LEN_MASK;
msg = tal_arr(NULL, u8, msglen);
Expand All @@ -66,9 +67,10 @@ int main(int argc, char *argv[])
!= crc32c(be32_to_cpu(hdr.timestamp), msg, msglen))
warnx("Checksum verification failed");

printf("%zu: %s%s", off,
printf("%zu: %s%s%s", off,
deleted ? "DELETED " : "",
push ? "PUSH " : "");
push ? "PUSH " : "",
ratelimit ? "RATE-LIMITED " : "");
if (deleted && !print_deleted) {
printf("\n");
goto end;
Expand Down
20 changes: 11 additions & 9 deletions gossipd/gossip_store.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -71,7 +71,7 @@ static ssize_t gossip_pwritev(int fd, const struct iovec *iov, int iovcnt,
#endif /* !HAVE_PWRITEV */

static bool append_msg(int fd, const u8 *msg, u32 timestamp,
bool push, u64 *len)
bool push, bool spam, u64 *len)
{
struct gossip_hdr hdr;
u32 msglen;
Expand All @@ -84,6 +84,8 @@ static bool append_msg(int fd, const u8 *msg, u32 timestamp,
hdr.len = cpu_to_be32(msglen);
if (push)
hdr.len |= CPU_TO_BE32(GOSSIP_STORE_LEN_PUSH_BIT);
if (spam)
hdr.len |= CPU_TO_BE32(GOSSIP_STORE_LEN_RATELIMIT_BIT);
hdr.crc = cpu_to_be32(crc32c(timestamp, msg, msglen));
hdr.timestamp = cpu_to_be32(timestamp);

Expand Down Expand Up @@ -118,7 +120,7 @@ static u8 *mk_private_channelmsg(const tal_t *ctx,
/* The upgrade from version 7 is trivial */
static bool can_upgrade(u8 oldversion)
{
return oldversion == 7 || oldversion == 8;
return oldversion == 7 || oldversion == 8 || oldversion == 9;
}

static bool upgrade_field(u8 oldversion,
Expand Down Expand Up @@ -277,7 +279,7 @@ static u32 gossip_store_compact_offline(struct routing_state *rstate)
oldlen = lseek(old_fd, SEEK_END, 0);
newlen = lseek(new_fd, SEEK_END, 0);
append_msg(old_fd, towire_gossip_store_ended(tmpctx, newlen),
0, true, &oldlen);
0, true, false, &oldlen);
close(old_fd);
status_debug("gossip_store_compact_offline: %zu deleted, %zu copied",
deleted, count);
Expand Down Expand Up @@ -565,7 +567,7 @@ bool gossip_store_compact(struct gossip_store *gs)

/* Write end marker now new one is ready */
append_msg(gs->fd, towire_gossip_store_ended(tmpctx, len),
0, true, &gs->len);
0, true, false, &gs->len);

gs->count = count;
gs->deleted = 0;
Expand All @@ -586,19 +588,19 @@ bool gossip_store_compact(struct gossip_store *gs)

u64 gossip_store_add(struct gossip_store *gs, const u8 *gossip_msg,
u32 timestamp, bool push,
const u8 *addendum)
bool spam, const u8 *addendum)
{
u64 off = gs->len;

/* Should never get here during loading! */
assert(gs->writable);

if (!append_msg(gs->fd, gossip_msg, timestamp, push, &gs->len)) {
if (!append_msg(gs->fd, gossip_msg, timestamp, push, spam, &gs->len)) {
status_broken("Failed writing to gossip store: %s",
strerror(errno));
return 0;
}
if (addendum && !append_msg(gs->fd, addendum, 0, false, &gs->len)) {
if (addendum && !append_msg(gs->fd, addendum, 0, false, false, &gs->len)) {
status_broken("Failed writing addendum to gossip store: %s",
strerror(errno));
return 0;
Expand All @@ -615,7 +617,7 @@ u64 gossip_store_add_private_update(struct gossip_store *gs, const u8 *update)
/* A local update for an unannounced channel: not broadcastable, but
* otherwise the same as a normal channel_update */
const u8 *pupdate = towire_gossip_store_private_update(tmpctx, update);
return gossip_store_add(gs, pupdate, 0, false, NULL);
return gossip_store_add(gs, pupdate, 0, false, false, NULL);
}

/* Returns index of following entry. */
Expand Down Expand Up @@ -675,7 +677,7 @@ void gossip_store_mark_channel_deleted(struct gossip_store *gs,
const struct short_channel_id *scid)
{
gossip_store_add(gs, towire_gossip_store_delete_chan(tmpctx, scid),
0, false, NULL);
0, false, false, NULL);
}

const u8 *gossip_store_get(const tal_t *ctx,
Expand Down
3 changes: 2 additions & 1 deletion gossipd/gossip_store.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -40,11 +40,12 @@ u64 gossip_store_add_private_update(struct gossip_store *gs, const u8 *update);
* @gossip_msg: the gossip message to insert.
* @timestamp: the timestamp for filtering of this messsage.
* @push: true if this should be sent to peers despite any timestamp filters.
* @spam: true if this message is rate-limited and squelched to peers.
* @addendum: another message to append immediately after this
* (for appending amounts to channel_announcements for internal use).
*/
u64 gossip_store_add(struct gossip_store *gs, const u8 *gossip_msg,
u32 timestamp, bool push, const u8 *addendum);
u32 timestamp, bool push, bool spam, const u8 *addendum);


/**
Expand Down
Loading