Skip to content

Commit 5115d66

Browse files
riptlriptl
committed
backtest: blockstore to pcap converter
Define a minimal pcapng-based file format to replace RocksDB as a shred source for `firedancer-dev backtest`. Add a fd_blockstore2shredcap tool to perform the conversion.
1 parent 7a8af28 commit 5115d66

File tree

7 files changed

+343
-14
lines changed

7 files changed

+343
-14
lines changed

‎src/discof/backtest/Local.mk‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
ifdef FD_HAS_ROCKSDB
22
$(call add-objs,fd_backtest_rocksdb fd_backtest_tile,fd_discof)
3+
$(call make-bin,fd_blockstore2shredcap,fd_blockstore2shredcap,fd_discof fd_flamenco fd_ballet fd_util,$(ROCKSDB_LIBS))
34
else
45
$(warning "rocksdb not installed, skipping backtest")
56
endif

‎src/discof/backtest/fd_backtest_rocksdb.c‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -124,6 +124,7 @@ int
124124
fd_backtest_rocksdb_next_root_slot( fd_backtest_rocksdb_t * db,
125125
ulong * root_slot,
126126
ulong * shred_cnt ) {
127+
FD_TEST( rocksdb_iter_valid( db->iter_root ) );
127128
rocksdb_iter_next( db->iter_root );
128129
if( FD_UNLIKELY( !rocksdb_iter_valid(db->iter_root) ) ) return 0;
129130

‎src/discof/backtest/fd_blockstore2shredcap.c‎

Lines changed: 201 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,201 @@
1+
#include "fd_backtest_rocksdb.h"
2+
#include "../../util/fd_util.h"
3+
#include "../../util/net/fd_pcapng.h"
4+
#include "../../util/net/fd_ip4.h"
5+
#include "../../util/net/fd_ip6.h"
6+
#include "fd_shredcap.h"
7+
#include "../../ballet/shred/fd_shred.h"
8+
#include "../../flamenco/gossip/fd_gossip_types.h"
9+
10+
#include <errno.h>
11+
#include <stdio.h>
12+
#include <stdlib.h>
13+
#include <fcntl.h>
14+
15+
/* Hardcoded constants */
16+
17+
#define IF_IDX_NET (0)
18+
#define IF_IDX_SHREDCAP (1)
19+
#define SHRED_PORT ((ushort)8003)
20+
21+
static int
22+
usage( int rc ) {
23+
fputs(
24+
"\n"
25+
"Usage: fd_blockstore2shredcap --rocksdb <path> --out <path>\n"
26+
"\n"
27+
"Extract rooted blocks from Agave RocksDB.\n"
28+
"Produces shredcap 0.1 (pcapng) file containing shreds and bank hashes.\n"
29+
"\n"
30+
" --rocksdb <path> Agave RocksDB directory\n"
31+
" --out <path> File path to new shredcap file (fails if file already exists)\n"
32+
"\n",
33+
stderr
34+
);
35+
return rc;
36+
}
37+
38+
static void
39+
write_endpoint( FILE * pcap ) {
40+
struct __attribute__((packed)) {
41+
uint type;
42+
fd_shredcap_endpoint_v0_t endpoint;
43+
} packet;
44+
memset( &packet, 0, sizeof(packet) );
45+
46+
packet.type = FD_SHREDCAP_TYPE_ENDPOINT_V0;
47+
fd_shredcap_endpoint_v0_t * endpoint = &packet.endpoint;
48+
fd_ip6_addr_ip4_mapped( endpoint->ip6_addr, FD_IP4_ADDR( 127,0,0,1 ) );
49+
endpoint->port = SHRED_PORT;
50+
endpoint->gossip_socket_type = FD_CONTACT_INFO_SOCKET_TVU;
51+
52+
fd_pcapng_fwrite_pkt1( pcap, &packet, sizeof(packet), IF_IDX_SHREDCAP, 0L );
53+
}
54+
55+
static void
56+
write_bank_hash( FILE * pcap,
57+
ulong slot,
58+
ulong shred_cnt,
59+
uchar const bank_hash[32] ) {
60+
struct __attribute__((packed)) {
61+
uint type;
62+
fd_shredcap_bank_hash_v0_t bank_hash_rec;
63+
} packet;
64+
memset( &packet, 0, sizeof(packet) );
65+
66+
packet.type = FD_SHREDCAP_TYPE_BANK_HASH_V0;
67+
fd_shredcap_bank_hash_v0_t * bank_hash_rec = &packet.bank_hash_rec;
68+
bank_hash_rec->slot = slot;
69+
bank_hash_rec->data_shred_cnt = shred_cnt;
70+
memcpy( bank_hash_rec->bank_hash, bank_hash, 32UL );
71+
72+
fd_pcapng_fwrite_pkt1( pcap, &packet, sizeof(packet), IF_IDX_SHREDCAP, 0L );
73+
}
74+
75+
static void
76+
write_shred( FILE * pcap,
77+
void const * shred ) {
78+
ulong shred_sz = fd_shred_sz( shred );
79+
FD_TEST( shred_sz<=FD_SHRED_MAX_SZ );
80+
81+
struct __attribute__((packed)) {
82+
fd_ip4_hdr_t ip4;
83+
fd_udp_hdr_t udp;
84+
uchar shred[ FD_SHRED_MAX_SZ ];
85+
} packet;
86+
87+
packet.ip4 = (fd_ip4_hdr_t) {
88+
.verihl = FD_IP4_VERIHL( 4, 5 ),
89+
.tos = 0,
90+
.net_tot_len = fd_ushort_bswap( (ushort)( 28+shred_sz ) ),
91+
.net_id = 0,
92+
.net_frag_off = fd_ushort_bswap( FD_IP4_HDR_FRAG_OFF_DF ),
93+
.ttl = 64,
94+
.protocol = FD_IP4_HDR_PROTOCOL_UDP,
95+
.check = 0,
96+
.saddr = FD_IP4_ADDR( 127,0,0,1 ),
97+
.daddr = FD_IP4_ADDR( 127,0,0,1 ),
98+
};
99+
packet.ip4.check = fd_ip4_hdr_check_fast( &packet.ip4 );
100+
packet.udp = (fd_udp_hdr_t) {
101+
.net_sport = fd_ushort_bswap( 42424 ),
102+
.net_dport = fd_ushort_bswap( SHRED_PORT ),
103+
.net_len = fd_ushort_bswap( (ushort)( 8+shred_sz ) ),
104+
.check = 0,
105+
};
106+
fd_memcpy( packet.shred, shred, shred_sz );
107+
108+
fd_pcapng_fwrite_pkt1( pcap, &packet, 28UL+shred_sz, IF_IDX_NET, 0L );
109+
}
110+
111+
int
112+
main( int argc,
113+
char ** argv ) {
114+
if( fd_env_strip_cmdline_contains( &argc, &argv, "--help" ) ) return usage( 0 );
115+
116+
char const * rocksdb_path = fd_env_strip_cmdline_cstr( &argc, &argv, "--rocksdb", NULL, NULL );
117+
char const * out_path = fd_env_strip_cmdline_cstr( &argc, &argv, "--out", NULL, NULL );
118+
char const * out_short = fd_env_strip_cmdline_cstr( &argc, &argv, "--o", NULL, NULL );
119+
if( !out_path ) out_path = out_short;
120+
121+
if( FD_UNLIKELY( !rocksdb_path ) ) {
122+
fputs( "Error: --rocksdb not specified\n", stderr );
123+
return usage( 1 );
124+
}
125+
if( FD_UNLIKELY( !out_path ) ) {
126+
fputs( "Error: --out not specified\n", stderr );
127+
return usage( 1 );
128+
}
129+
130+
fd_boot( &argc, &argv );
131+
132+
void * rocks_mem = aligned_alloc( fd_backtest_rocksdb_align(), fd_backtest_rocksdb_footprint() );
133+
if( FD_UNLIKELY( !rocks_mem ) ) FD_LOG_ERR(( "out of memory" ));
134+
fd_backtest_rocksdb_t * rocksdb = fd_backtest_rocksdb_join( fd_backtest_rocksdb_new( rocks_mem, rocksdb_path ) );
135+
if( FD_UNLIKELY( !rocksdb ) ) FD_LOG_ERR(( "failed to open RocksDB at %s", rocksdb_path ));
136+
fd_backtest_rocksdb_init( rocksdb, 0UL );
137+
138+
int out_fd = open( out_path, O_WRONLY|O_CREAT|O_EXCL, 0644 );
139+
if( FD_UNLIKELY( out_fd<0 ) ) FD_LOG_ERR(( "failed to create file %s (%i-%s)", out_path, errno, fd_io_strerror( errno ) ));
140+
FILE * out = fdopen( out_fd, "wb" );
141+
if( FD_UNLIKELY( !out ) ) FD_LOG_ERR(( "fdopen failed on %s (%i-%s)", out_path, errno, fd_io_strerror( errno ) ));
142+
143+
/* Write pcapng header */
144+
{
145+
fd_pcapng_shb_opts_t shb_opts;
146+
fd_pcapng_shb_defaults( &shb_opts );
147+
if( FD_UNLIKELY( !fd_pcapng_fwrite_shb( &shb_opts, out ) ) ) FD_LOG_ERR(( "pcap write error" ));
148+
}
149+
uint idb_cnt = 0U;
150+
{
151+
fd_pcapng_idb_opts_t idb_opts = {
152+
.name = "lo",
153+
.ip4_addr = { 127,0,0,1 }
154+
};
155+
if( FD_UNLIKELY( !fd_pcapng_fwrite_idb( FD_PCAPNG_LINKTYPE_IPV4, &idb_opts, out ) ) ) FD_LOG_ERR(( "pcap write error" ));
156+
FD_TEST( idb_cnt++==IF_IDX_NET );
157+
}
158+
{
159+
fd_pcapng_idb_opts_t idb_opts = {
160+
.name = "shredcap0",
161+
};
162+
if( FD_UNLIKELY( !fd_pcapng_fwrite_idb( FD_PCAPNG_LINKTYPE_USER0, &idb_opts, out ) ) ) FD_LOG_ERR(( "pcap write error" ));
163+
FD_TEST( idb_cnt++==IF_IDX_SHREDCAP );
164+
}
165+
write_endpoint( out );
166+
167+
ulong slot_cnt = 0UL;
168+
for( ;; slot_cnt++ ) {
169+
170+
ulong root_slot;
171+
ulong shred_cnt;
172+
int root_ok = fd_backtest_rocksdb_next_root_slot( rocksdb, &root_slot, &shred_cnt );
173+
if( !root_ok ) break;
174+
uchar const * bank_hash = fd_backtest_rocksdb_bank_hash( rocksdb, root_slot );
175+
if( FD_UNLIKELY( !bank_hash ) ) FD_LOG_ERR(( "failed to extract bank hash for root slot %lu", root_slot ));
176+
177+
write_bank_hash( out, root_slot, shred_cnt, bank_hash );
178+
179+
for( ulong i=0UL; i<shred_cnt; i++ ) {
180+
void const * shred = fd_backtest_rocksdb_shred( rocksdb, root_slot, i );
181+
if( FD_UNLIKELY( !shred ) ) {
182+
FD_LOG_WARNING(( "missing shred %lu for slot %lu", i, root_slot ));
183+
break;
184+
}
185+
write_shred( out, shred );
186+
}
187+
188+
}
189+
190+
long off = ftell( out );
191+
FD_LOG_NOTICE(( "%s: wrote %lu slots, %ld bytes", out_path, slot_cnt, off ));
192+
193+
/* FIXME missing destructor for backtest_rocksdb */
194+
if( FD_UNLIKELY( 0!=fclose( out ) ) ) {
195+
FD_LOG_ERR(( "fclose failed on %s (%i-%s), output file may be corrupt", out_path, errno, fd_io_strerror( errno ) ));
196+
}
197+
free( rocks_mem );
198+
199+
fd_halt();
200+
return 0;
201+
}

‎src/discof/backtest/fd_shredcap.h‎

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,28 @@
1+
#ifndef HEADER_fd_src_discof_backtest_fd_shredcap_h
2+
#define HEADER_fd_src_discof_backtest_fd_shredcap_h
3+
4+
/* fd_shredcap.h provides C definitions for shredcap v0.1 file format
5+
bits. */
6+
7+
#include "../../util/fd_util_base.h"
8+
9+
#define FD_SHREDCAP_V0_IFNAME "shredcap0"
10+
11+
#define FD_SHREDCAP_TYPE_BANK_HASH_V0 (0x1u)
12+
#define FD_SHREDCAP_TYPE_ENDPOINT_V0 (0x2u)
13+
14+
struct __attribute__((packed)) fd_shredcap_bank_hash_v0 {
15+
ulong slot;
16+
uchar bank_hash[32];
17+
ulong data_shred_cnt;
18+
};
19+
typedef struct fd_shredcap_bank_hash_v0 fd_shredcap_bank_hash_v0_t;
20+
21+
struct __attribute__((packed)) fd_shredcap_endpoint_v0 {
22+
uchar ip6_addr[16]; /* net order */
23+
ushort port; /* little endian */
24+
uint gossip_socket_type; /* little endian */
25+
};
26+
typedef struct fd_shredcap_endpoint_v0 fd_shredcap_endpoint_v0_t;
27+
28+
#endif /* HEADER_fd_src_discof_backtest_fd_shredcap_h */

‎src/discof/backtest/shredcap.md‎

Lines changed: 93 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,93 @@
1+
# shredcap v0.1 file format
2+
3+
The *shredcap* container holds Solana block data captures suitable for
4+
replay in a streaming file format.
5+
6+
This document specifies the shredcap v0.1 file format.
7+
8+
## Container
9+
10+
Shredcap is layered on top of [pcapng](https://pcapng.com/). Shredcap
11+
flows are designed to co-exist with unrelated packet types, such as
12+
regular network traffic.
13+
14+
At a high level, shredcap stores block data via UDP/IP packets. Each
15+
packet contains a shred in Solana wire format (Turbine). Readers
16+
require context to identify and interpret shred packets. This context
17+
is embedded in the form of [interfaces](#interfaces) and
18+
[metadata packets](#metadata-packets).
19+
20+
Readers _should_ transparently handle files with multiple Section Header
21+
Blocks (SHB) by resetting any parse state such as cached interfaces
22+
or endpoints.
23+
24+
## Interfaces
25+
26+
### Network Interface
27+
28+
Each pcapng section in a shredcap file MUST contain at least one network
29+
interface.
30+
31+
A network interface is identified by an Interface Description Block
32+
(IDB) with link type `LINKTYPE_ETHERNET` (1), `LINKTYPE_RAW` (101), or
33+
`LINKTYPE_IPV4` (228).
34+
35+
### Metadata Interface
36+
37+
Each pcapng section in a shredcap file MUST contain exactly one
38+
IDB with the `if_name` option (2) set to the string `shredcap0`.
39+
This IDB is referred to as the *metadata interface*.
40+
41+
## Packet Records
42+
43+
Enhanced Packet Blocks (EPB) may contain shredcap data.
44+
Simple Packet Blocks (SPB) are considered obsolete and MUST be ignored.
45+
46+
### Metadata Packets
47+
48+
Metadata packets are EPBs where the *interface ID* refers to a
49+
[metadata interface](#metadata-interface).
50+
51+
The format for shredcap v0.1 metadata packets is as follows. All fixed
52+
with integer types are encoded in little endian unless otherwise
53+
specified.
54+
55+
- Metadata Type (uint32)
56+
- Type-specific data (variable size)
57+
58+
#### Bank hash v0
59+
60+
Root slot metadata encodes state transition information after replaying
61+
a Solana block.
62+
63+
The format for shredcap v0.1 bank hash v0 metadata is as follows:
64+
65+
- Metadata Type: `0x1` (bank hash v0)
66+
- Slot number (uint64)
67+
- Bank hash (32 bytes): Bank hash seen after replaying a slot
68+
- Data shred count (uint64): The number of data shreds with block data
69+
ingested to produce this bank hash
70+
71+
#### Endpoint v0
72+
73+
The format for shredcap v0.1 endpoint v0 metadata is as follows:
74+
75+
- Metadata Type: `0x2` (endpoint v0)
76+
- IPv6 address (16 bytes)
77+
- UDP port number (uint16)
78+
- Gossip socket type (uint32): `0xa` for shreds
79+
80+
Note that IPv4 addresses are encoded as IPv4-mapped IPv6 addresses.
81+
82+
## UDP Packets
83+
84+
UDP packets are EPBs are identified as follows:
85+
- the *interface ID* refers to a network interface
86+
- the packet contains an IPv4 header
87+
- the packet contains an UDP header
88+
89+
### Shred
90+
91+
Shred packets are [UDP packets](#udp-packets) whose IPv4 destination
92+
address and UDP destination port map to an [endpoint](#endpoint) with
93+
gossip socket type `0xa`.

‎src/util/net/fd_pcapng.c‎

Lines changed: 6 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -653,10 +653,11 @@ fd_pcapng_fwrite_idb( uint link_type,
653653
}
654654

655655
ulong
656-
fd_pcapng_fwrite_pkt( long ts,
657-
void const * payload,
658-
ulong payload_sz,
659-
void * _file ) {
656+
fd_pcapng_fwrite_pkt1( void * _file,
657+
void const * payload,
658+
ulong payload_sz,
659+
uint if_idx,
660+
long ts ) {
660661

661662
FILE * file = (FILE *)_file;
662663
FD_TEST( fd_ulong_is_aligned( (ulong)ftell( file ), 4UL ) );
@@ -665,7 +666,7 @@ fd_pcapng_fwrite_pkt( long ts,
665666
fd_pcapng_epb_t block = {
666667
.block_type = FD_PCAPNG_BLOCK_TYPE_EPB,
667668
/* block_sz set later */
668-
.if_idx = 0U,
669+
.if_idx = if_idx,
669670
.ts_hi = (uint)( (ulong)ts >> 32UL ),
670671
.ts_lo = (uint)( (ulong)ts ),
671672
.cap_len = (uint)payload_sz,

0 commit comments

Comments
 (0)