Skip to content

replay, sched: verify PoH #7547

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
yufeng-jump wants to merge 1 commit into
base: main
Choose a base branch
from
Open

replay, sched: verify PoH #7547

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions src/app/firedancer/config/default.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -766,7 +766,7 @@ user = ""
# much parallelism is possible.
#
# More exec tiles can allow the validator to replay through blocks
# faster, subject to Amdahl's law. Empirically, we've found 8 exec
# faster, subject to Amdahl's law. Empirically, we've found 10 exec
# tiles to achieve near optimal replay speed under current
# `mainnet-beta` conditions.
#
Expand All @@ -775,7 +775,7 @@ user = ""
# to be highly concurrent, most of the time account writeback can be
# done in parallel without blocking. Multiple exec tiles exploit this
# parallelism supported by the accounts database.
exec_tile_count = 8
exec_tile_count = 10

# How many shred tiles to run. Should be set to 1. This is
# configurable and designed to scale out for future network
Expand Down
16 changes: 14 additions & 2 deletions src/discof/exec/fd_exec_tile.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@
#include "generated/fd_exec_tile_seccomp.h"

#include "../../util/pod/fd_pod_format.h"
#include "../../ballet/sha256/fd_sha256.h" /* fd_sha256_hash_32_repeated */
#include "../../discof/replay/fd_exec.h"
#include "../../flamenco/runtime/context/fd_capture_ctx.h"
#include "../../flamenco/runtime/fd_bank.h"
Expand Down Expand Up @@ -128,7 +129,7 @@ returnable_frag( fd_exec_tile_ctx_t * ctx,
fd_exec_txn_exec_msg_t * msg = fd_chunk_to_laddr( ctx->replay_in->mem, chunk );
ctx->bank = fd_banks_bank_query( ctx->banks, msg->bank_idx );
FD_TEST( ctx->bank );
ctx->txn_in.txn = &msg->txn;
ctx->txn_in.txn = msg->txn;
ctx->txn_in.exec_accounts = &ctx->exec_accounts;

fd_runtime_prepare_and_execute_txn( ctx->runtime, ctx->bank, &ctx->txn_in, &ctx->txn_out );
Expand Down Expand Up @@ -158,7 +159,7 @@ returnable_frag( fd_exec_tile_ctx_t * ctx,
}
case FD_EXEC_TT_TXN_SIGVERIFY: {
fd_exec_txn_sigverify_msg_t * msg = fd_chunk_to_laddr( ctx->replay_in->mem, chunk );
int res = fd_executor_txn_verify( &msg->txn, ctx->sha_lj );
int res = fd_executor_txn_verify( msg->txn, ctx->sha_lj );
fd_exec_task_done_msg_t * out_msg = fd_chunk_to_laddr( ctx->exec_replay_out->mem, ctx->exec_replay_out->chunk );
out_msg->bank_idx = msg->bank_idx;
out_msg->txn_sigverify->txn_idx = msg->txn_idx;
Expand All @@ -167,6 +168,17 @@ returnable_frag( fd_exec_tile_ctx_t * ctx,
ctx->exec_replay_out->chunk = fd_dcache_compact_next( ctx->exec_replay_out->chunk, sizeof(*out_msg), ctx->exec_replay_out->chunk0, ctx->exec_replay_out->wmark );
break;
}
case FD_EXEC_TT_POH_HASH: {
fd_exec_poh_hash_msg_t * msg = fd_chunk_to_laddr( ctx->replay_in->mem, chunk );
fd_exec_task_done_msg_t * out_msg = fd_chunk_to_laddr( ctx->exec_replay_out->mem, ctx->exec_replay_out->chunk );
out_msg->bank_idx = msg->bank_idx;
out_msg->poh_hash->mblk_idx = msg->mblk_idx;
out_msg->poh_hash->hashcnt = msg->hashcnt;
fd_sha256_hash_32_repeated( msg->hash, out_msg->poh_hash->hash, msg->hashcnt );
fd_stem_publish( stem, ctx->exec_replay_out->idx, (FD_EXEC_TT_POH_HASH<<32)|ctx->tile_idx, ctx->exec_replay_out->chunk, sizeof(*out_msg), 0UL, 0UL, 0UL );
ctx->exec_replay_out->chunk = fd_dcache_compact_next( ctx->exec_replay_out->chunk, sizeof(*out_msg), ctx->exec_replay_out->chunk0, ctx->exec_replay_out->wmark );
break;
}
default: FD_LOG_CRIT(( "unexpected signature %lu", sig ));
}
} else FD_LOG_CRIT(( "invalid in_idx %lu", in_idx ));
Expand Down
23 changes: 20 additions & 3 deletions src/discof/replay/fd_exec.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@
#define FD_EXEC_TT_TXN_EXEC (1UL) /* Transaction execution. */
#define FD_EXEC_TT_TXN_SIGVERIFY (2UL) /* Transaction sigverify. */
#define FD_EXEC_TT_LTHASH (3UL) /* Account lthash. */
#define FD_EXEC_TT_POH_VERIFY (4UL) /* PoH hash verification. */
#define FD_EXEC_TT_POH_HASH (4UL) /* PoH hashing. */

/* Sent from the replay tile to the exec tiles. These describe one of
several types of tasks for an exec tile. An idx to the bank in the
Expand All @@ -20,20 +20,29 @@
struct fd_exec_txn_exec_msg {
ulong bank_idx;
ulong txn_idx;
fd_txn_p_t txn;
fd_txn_p_t txn[ 1 ];
};
typedef struct fd_exec_txn_exec_msg fd_exec_txn_exec_msg_t;

struct fd_exec_txn_sigverify_msg {
ulong bank_idx;
ulong txn_idx;
fd_txn_p_t txn;
fd_txn_p_t txn[ 1 ];
};
typedef struct fd_exec_txn_sigverify_msg fd_exec_txn_sigverify_msg_t;

struct fd_exec_poh_hash_msg {
ulong bank_idx;
ulong mblk_idx;
ulong hashcnt;
fd_hash_t hash[ 1 ];
};
typedef struct fd_exec_poh_hash_msg fd_exec_poh_hash_msg_t;

union fd_exec_task_msg {
fd_exec_txn_exec_msg_t txn_exec;
fd_exec_txn_sigverify_msg_t txn_sigverify;
fd_exec_poh_hash_msg_t poh_hash;
};
typedef union fd_exec_task_msg fd_exec_task_msg_t;

Expand All @@ -59,11 +68,19 @@ struct fd_exec_txn_sigverify_done_msg {
};
typedef struct fd_exec_txn_sigverify_done_msg fd_exec_txn_sigverify_done_msg_t;

struct fd_exec_poh_hash_done_msg {
ulong mblk_idx;
ulong hashcnt;
fd_hash_t hash[ 1 ];
};
typedef struct fd_exec_poh_hash_done_msg fd_exec_poh_hash_done_msg_t;

struct fd_exec_task_done_msg {
ulong bank_idx;
union {
fd_exec_txn_exec_done_msg_t txn_exec[ 1 ];
fd_exec_txn_sigverify_done_msg_t txn_sigverify[ 1 ];
fd_exec_poh_hash_done_msg_t poh_hash[ 1 ];
};
};
typedef struct fd_exec_task_done_msg fd_exec_task_done_msg_t;
Expand Down
63 changes: 47 additions & 16 deletions src/discof/replay/fd_replay_tile.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -603,22 +603,21 @@ replay_block_start( fd_replay_tile_t * ctx,

fd_bank_has_identity_vote_set( bank, 0 );

/* Set the tick height. */
fd_bank_tick_height_set( bank, fd_bank_max_tick_height_get( bank ) );

/* Update block height. */
fd_bank_block_height_set( bank, fd_bank_block_height_get( bank ) + 1UL );

ulong * max_tick_height = fd_bank_max_tick_height_modify( bank );
ulong ticks_per_slot = fd_bank_ticks_per_slot_get( bank );
if( FD_UNLIKELY( FD_RUNTIME_EXECUTE_SUCCESS != fd_runtime_compute_max_tick_height( ticks_per_slot, slot, max_tick_height ) ) ) {
FD_LOG_CRIT(( "couldn't compute tick height/max tick height slot %lu ticks_per_slot %lu", slot, ticks_per_slot ));
}

int is_epoch_boundary = 0;
fd_runtime_block_execute_prepare( ctx->banks, bank, ctx->accdb, &ctx->runtime_stack, ctx->capture_ctx, &is_epoch_boundary );
if( FD_UNLIKELY( is_epoch_boundary ) ) publish_stake_weights( ctx, stem, bank, 1 );

ulong max_tick_height;
if( FD_UNLIKELY( FD_RUNTIME_EXECUTE_SUCCESS!=fd_runtime_compute_max_tick_height( fd_bank_ticks_per_slot_get( parent_bank ), slot, &max_tick_height ) ) ) {
FD_LOG_CRIT(( "couldn't compute tick height/max tick height slot %lu ticks_per_slot %lu", slot, fd_bank_ticks_per_slot_get( bank ) ));
}
fd_bank_max_tick_height_set( bank, max_tick_height );
fd_bank_tick_height_set( bank, fd_bank_max_tick_height_get( parent_bank ) ); /* The parent's max tick height is our starting tick height. */
fd_sched_set_poh_params( ctx->sched, bank->idx, fd_bank_tick_height_get( bank ), fd_bank_max_tick_height_get( bank ), fd_bank_hashes_per_tick_get( bank ), fd_bank_poh_query( parent_bank ) );

return bank;
}

Expand Down Expand Up @@ -1525,7 +1524,7 @@ dispatch_task( fd_replay_tile_t * ctx,

fd_replay_out_link_t * exec_out = ctx->exec_out;
fd_exec_txn_exec_msg_t * exec_msg = fd_chunk_to_laddr( exec_out->mem, exec_out->chunk );
memcpy( &exec_msg->txn, txn_p, sizeof(fd_txn_p_t) );
memcpy( exec_msg->txn, txn_p, sizeof(fd_txn_p_t) );
exec_msg->bank_idx = task->txn_exec->bank_idx;
exec_msg->txn_idx = task->txn_exec->txn_idx;
fd_stem_publish( stem, exec_out->idx, (FD_EXEC_TT_TXN_EXEC<<32) | task->txn_exec->exec_idx, exec_out->chunk, sizeof(*exec_msg), 0UL, 0UL, fd_frag_meta_ts_comp( fd_tickcount() ) );
Expand All @@ -1540,13 +1539,27 @@ dispatch_task( fd_replay_tile_t * ctx,

fd_replay_out_link_t * exec_out = ctx->exec_out;
fd_exec_txn_sigverify_msg_t * exec_msg = fd_chunk_to_laddr( exec_out->mem, exec_out->chunk );
memcpy( &exec_msg->txn, txn_p, sizeof(fd_txn_p_t) );
memcpy( exec_msg->txn, txn_p, sizeof(fd_txn_p_t) );
exec_msg->bank_idx = task->txn_sigverify->bank_idx;
exec_msg->txn_idx = task->txn_sigverify->txn_idx;
fd_stem_publish( stem, exec_out->idx, (FD_EXEC_TT_TXN_SIGVERIFY<<32) | task->txn_sigverify->exec_idx, exec_out->chunk, sizeof(*exec_msg), 0UL, 0UL, 0UL );
exec_out->chunk = fd_dcache_compact_next( exec_out->chunk, sizeof(*exec_msg), exec_out->chunk0, exec_out->wmark );
break;
};
case FD_SCHED_TT_POH_HASH: {
fd_bank_t * bank = fd_banks_bank_query( ctx->banks, task->poh_hash->bank_idx );
bank->refcnt++;

fd_replay_out_link_t * exec_out = ctx->exec_out;
fd_exec_poh_hash_msg_t * exec_msg = fd_chunk_to_laddr( exec_out->mem, exec_out->chunk );
exec_msg->bank_idx = task->poh_hash->bank_idx;
exec_msg->mblk_idx = task->poh_hash->mblk_idx;
exec_msg->hashcnt = task->poh_hash->hashcnt;
memcpy( exec_msg->hash, task->poh_hash->hash, sizeof(fd_hash_t) );
fd_stem_publish( stem, exec_out->idx, (FD_EXEC_TT_POH_HASH<<32) | task->poh_hash->exec_idx, exec_out->chunk, sizeof(*exec_msg), 0UL, 0UL, 0UL );
exec_out->chunk = fd_dcache_compact_next( exec_out->chunk, sizeof(*exec_msg), exec_out->chunk0, exec_out->wmark );
break;
};
default: {
FD_LOG_CRIT(( "unexpected task type %lu", task->task_type ));
}
Expand All @@ -1571,22 +1584,29 @@ replay( fd_replay_tile_t * ctx,
switch( task->task_type ) {
case FD_SCHED_TT_BLOCK_START: {
replay_block_start( ctx, stem, task->block_start->bank_idx, task->block_start->parent_bank_idx, task->block_start->slot );
fd_sched_task_done( ctx->sched, FD_SCHED_TT_BLOCK_START, ULONG_MAX, ULONG_MAX );
fd_sched_task_done( ctx->sched, FD_SCHED_TT_BLOCK_START, ULONG_MAX, ULONG_MAX, NULL );
break;
}
case FD_SCHED_TT_BLOCK_END: {
fd_bank_t * bank = fd_banks_bank_query( ctx->banks, task->block_end->bank_idx );
if( FD_LIKELY( !(bank->flags&FD_BANK_FLAGS_DEAD) ) ) replay_block_finalize( ctx, stem, bank );
fd_sched_task_done( ctx->sched, FD_SCHED_TT_BLOCK_END, ULONG_MAX, ULONG_MAX );
fd_sched_task_done( ctx->sched, FD_SCHED_TT_BLOCK_END, ULONG_MAX, ULONG_MAX, NULL );
break;
}
case FD_SCHED_TT_TXN_EXEC:
case FD_SCHED_TT_TXN_SIGVERIFY: {
case FD_SCHED_TT_TXN_SIGVERIFY:
case FD_SCHED_TT_POH_HASH: {
/* Likely/common case: we have a transaction we actually need to
execute. */
dispatch_task( ctx, stem, task );
break;
}
case FD_SCHED_TT_MARK_DEAD: {
fd_bank_t * bank = fd_banks_bank_query( ctx->banks, task->mark_dead->bank_idx );
publish_slot_dead( ctx, stem, bank );
fd_banks_mark_bank_dead( ctx->banks, bank );
break;
}
default: {
FD_LOG_CRIT(( "unexpected task type %lu", task->task_type ));
}
Expand Down Expand Up @@ -1976,7 +1996,8 @@ process_exec_task_done( fd_replay_tile_t * ctx,
if( FD_UNLIKELY( (bank->flags&FD_BANK_FLAGS_DEAD) && bank->refcnt==0UL ) ) {
fd_banks_mark_bank_frozen( ctx->banks, bank );
}
fd_sched_task_done( ctx->sched, FD_SCHED_TT_TXN_EXEC, msg->txn_exec->txn_idx, exec_tile_idx );
int res = fd_sched_task_done( ctx->sched, FD_SCHED_TT_TXN_EXEC, msg->txn_exec->txn_idx, exec_tile_idx, NULL );
FD_TEST( res==0 );
break;
}
case FD_EXEC_TT_TXN_SIGVERIFY: {
Expand All @@ -1991,7 +2012,17 @@ process_exec_task_done( fd_replay_tile_t * ctx,
if( FD_UNLIKELY( (bank->flags&FD_BANK_FLAGS_DEAD) && bank->refcnt==0UL ) ) {
fd_banks_mark_bank_frozen( ctx->banks, bank );
}
fd_sched_task_done( ctx->sched, FD_SCHED_TT_TXN_SIGVERIFY, msg->txn_sigverify->txn_idx, exec_tile_idx );
int res = fd_sched_task_done( ctx->sched, FD_SCHED_TT_TXN_SIGVERIFY, msg->txn_sigverify->txn_idx, exec_tile_idx, NULL );
FD_TEST( res==0 );
break;
}
case FD_EXEC_TT_POH_HASH: {
int res = fd_sched_task_done( ctx->sched, FD_SCHED_TT_POH_HASH, ULONG_MAX, exec_tile_idx, msg->poh_hash );
if( FD_UNLIKELY( res<0 && !(bank->flags&FD_BANK_FLAGS_DEAD) ) ) {
publish_slot_dead( ctx, stem, bank );
fd_banks_mark_bank_dead( ctx->banks, bank );
}
if( FD_UNLIKELY( (bank->flags&FD_BANK_FLAGS_DEAD) && bank->refcnt==0UL ) ) fd_banks_mark_bank_frozen( ctx->banks, bank );
break;
}
default: FD_LOG_CRIT(( "unexpected sig 0x%lx", sig ));
Expand Down
Loading
Loading