Skip to content

feat(fortuna): add metrics for retry count, gas/fee multipliers, and fee estimates #2258

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 2 commits into from
Jan 18, 2025
Merged

feat(fortuna): add metrics for retry count, gas/fee multipliers, and fee estimates #2258

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
2011d30
feat(fortuna): add metrics for retry count, gas/fee multipliers, and …
devin-ai-integration[bot] Jan 15, 2025
617d0f9
cleanup
jayantk Jan 17, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion apps/fortuna/Cargo.lock
View file
Open in desktop

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

2 changes: 1 addition & 1 deletion apps/fortuna/Cargo.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
[package]
name = "fortuna"
version = "7.2.0"
version = "7.3.0"
edition = "2021"

[dependencies]
Expand Down
91 changes: 91 additions & 0 deletions apps/fortuna/src/keeper.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -69,6 +69,7 @@ pub struct KeeperMetrics {
pub balance: Family<AccountLabel, Gauge<f64, AtomicU64>>,
pub collected_fee: Family<AccountLabel, Gauge<f64, AtomicU64>>,
pub current_fee: Family<AccountLabel, Gauge<f64, AtomicU64>>,
pub target_provider_fee: Family<AccountLabel, Gauge<f64, AtomicU64>>,
pub total_gas_spent: Family<AccountLabel, Gauge<f64, AtomicU64>>,
pub total_gas_fee_spent: Family<AccountLabel, Gauge<f64, AtomicU64>>,
pub requests: Family<AccountLabel, Counter>,
Expand All @@ -78,6 +79,10 @@ pub struct KeeperMetrics {
pub requests_reprocessed: Family<AccountLabel, Counter>,
pub reveals: Family<AccountLabel, Counter>,
pub request_duration_ms: Family<AccountLabel, Histogram>,
pub retry_count: Family<AccountLabel, Histogram>,
pub final_gas_multiplier: Family<AccountLabel, Histogram>,
pub final_fee_multiplier: Family<AccountLabel, Histogram>,
pub gas_price_estimate: Family<AccountLabel, Gauge<f64, AtomicU64>>,
}

impl Default for KeeperMetrics {
Expand All @@ -88,6 +93,7 @@ impl Default for KeeperMetrics {
balance: Family::default(),
collected_fee: Family::default(),
current_fee: Family::default(),
target_provider_fee: Family::default(),
total_gas_spent: Family::default(),
total_gas_fee_spent: Family::default(),
requests: Family::default(),
Expand All @@ -105,6 +111,18 @@ impl Default for KeeperMetrics {
.into_iter(),
)
}),
retry_count: Family::new_with_constructor(|| {
Histogram::new(vec![0.0, 1.0, 2.0, 3.0, 4.0, 5.0, 10.0, 15.0, 20.0].into_iter())
}),
final_gas_multiplier: Family::new_with_constructor(|| {
Histogram::new(
vec![100.0, 125.0, 150.0, 200.0, 300.0, 400.0, 500.0, 600.0].into_iter(),
)
}),
final_fee_multiplier: Family::new_with_constructor(|| {
Histogram::new(vec![100.0, 110.0, 120.0, 140.0, 160.0, 180.0, 200.0].into_iter())
}),
gas_price_estimate: Family::default(),
}
}
}
Expand Down Expand Up @@ -174,6 +192,12 @@ impl KeeperMetrics {
keeper_metrics.current_fee.clone(),
);

writable_registry.register(
"target_provider_fee",
"Target fee in ETH -- differs from current_fee in that this is the goal, and current_fee is the on-chain value.",
keeper_metrics.target_provider_fee.clone(),
);

writable_registry.register(
"total_gas_spent",
"Total gas spent revealing requests",
Expand All @@ -198,6 +222,30 @@ impl KeeperMetrics {
keeper_metrics.request_duration_ms.clone(),
);

writable_registry.register(
"retry_count",
"Number of retries for successful transactions",
keeper_metrics.retry_count.clone(),
);

writable_registry.register(
"final_gas_multiplier",
"Final gas multiplier percentage for successful transactions",
keeper_metrics.final_gas_multiplier.clone(),
);

writable_registry.register(
"final_fee_multiplier",
"Final fee multiplier percentage for successful transactions",
keeper_metrics.final_fee_multiplier.clone(),
);

writable_registry.register(
"gas_price_estimate",
"Gas price estimate for the blockchain (in gwei)",
keeper_metrics.gas_price_estimate.clone(),
);

keeper_metrics
}
}
Expand Down Expand Up @@ -327,6 +375,7 @@ pub async fn run_keeper_threads(
spawn(
adjust_fee_wrapper(
contract.clone(),
chain_state.clone(),
chain_state.provider_address,
ADJUST_FEE_INTERVAL,
chain_eth_config.legacy_tx,
Expand All @@ -346,6 +395,7 @@ pub async fn run_keeper_threads(
u64::try_from(100 + chain_eth_config.max_profit_pct)
.expect("max_profit_pct must be >= -100"),
chain_eth_config.fee,
metrics.clone(),
)
.in_current_span(),
);
Expand Down Expand Up @@ -479,6 +529,25 @@ pub async fn process_event_with_backoff(
.request_duration_ms
.get_or_create(&account_label)
.observe(duration.as_millis() as f64);

// Track retry count, gas multiplier, and fee multiplier for successful transactions
let num_retries = num_retries.load(std::sync::atomic::Ordering::Relaxed);
metrics
.retry_count
.get_or_create(&account_label)
.observe(num_retries as f64);

let gas_multiplier = escalation_policy.get_gas_multiplier_pct(num_retries);
metrics
.final_gas_multiplier
.get_or_create(&account_label)
.observe(gas_multiplier as f64);

let fee_multiplier = escalation_policy.get_fee_multiplier_pct(num_retries);
metrics
.final_fee_multiplier
.get_or_create(&account_label)
.observe(fee_multiplier as f64);
}
Err(e) => {
// In case the callback did not succeed, we double-check that the request is still on-chain.
Expand Down Expand Up @@ -1133,6 +1202,7 @@ pub async fn send_and_confirm(contract_call: PythContractCall) -> Result<()> {
#[allow(clippy::too_many_arguments)]
pub async fn adjust_fee_wrapper(
contract: Arc<InstrumentedSignablePythContract>,
chain_state: BlockchainState,
provider_address: Address,
poll_interval: Duration,
legacy_tx: bool,
Expand All @@ -1141,6 +1211,7 @@ pub async fn adjust_fee_wrapper(
target_profit_pct: u64,
max_profit_pct: u64,
min_fee_wei: u128,
metrics: Arc<KeeperMetrics>,
) {
// The maximum balance of accrued fees + provider wallet balance. None if we haven't observed a value yet.
let mut high_water_pnl: Option<U256> = None;
Expand All @@ -1149,6 +1220,7 @@ pub async fn adjust_fee_wrapper(
loop {
if let Err(e) = adjust_fee_if_necessary(
contract.clone(),
chain_state.id.clone(),
provider_address,
legacy_tx,
gas_limit,
Expand All @@ -1158,6 +1230,7 @@ pub async fn adjust_fee_wrapper(
min_fee_wei,
&mut high_water_pnl,
&mut sequence_number_of_last_fee_update,
metrics.clone(),
)
.in_current_span()
.await
Expand Down Expand Up @@ -1232,6 +1305,7 @@ pub async fn update_commitments_if_necessary(
#[allow(clippy::too_many_arguments)]
pub async fn adjust_fee_if_necessary(
contract: Arc<InstrumentedSignablePythContract>,
chain_id: ChainId,
provider_address: Address,
legacy_tx: bool,
gas_limit: u64,
Expand All @@ -1241,6 +1315,7 @@ pub async fn adjust_fee_if_necessary(
min_fee_wei: u128,
high_water_pnl: &mut Option<U256>,
sequence_number_of_last_fee_update: &mut Option<u64>,
metrics: Arc<KeeperMetrics>,
) -> Result<()> {
let provider_info = contract
.get_provider_info(provider_address)
Expand All @@ -1256,6 +1331,17 @@ pub async fn adjust_fee_if_necessary(
let max_callback_cost: u128 = estimate_tx_cost(contract.clone(), legacy_tx, gas_limit.into())
.await
.map_err(|e| anyhow!("Could not estimate transaction cost. error {:?}", e))?;

let account_label = AccountLabel {
chain_id: chain_id.clone(),
address: provider_address.to_string(),
};

metrics
.gas_price_estimate
.get_or_create(&account_label)
.set((max_callback_cost / u128::from(gas_limit)) as f64 / 1e9);

let target_fee_min = std::cmp::max(
(max_callback_cost * u128::from(min_profit_pct)) / 100,
min_fee_wei,
Expand All @@ -1264,6 +1350,11 @@ pub async fn adjust_fee_if_necessary(
(max_callback_cost * u128::from(target_profit_pct)) / 100,
min_fee_wei,
);
metrics
.target_provider_fee
.get_or_create(&account_label)
.set(((max_callback_cost * u128::from(target_profit_pct)) / 100) as f64 / 1e18);

let target_fee_max = std::cmp::max(
(max_callback_cost * u128::from(max_profit_pct)) / 100,
min_fee_wei,
Expand Down
Loading