diff --git a/core/tracker/reason.go b/core/tracker/reason.go index fa79403e6..523e589f7 100644 --- a/core/tracker/reason.go +++ b/core/tracker/reason.go @@ -19,136 +19,118 @@ var ( Long: "Reason `unknown` indicates an unknown error occurred.", } - reasonFetcherBN = reason{ - Code: "fetcher_bn", + reasonFetchBNError = reason{ + Code: "fetch_bn_error", Short: "couldn't fetch duty data from the beacon node", - Long: "Reason `fetcher_bn` indicates a duty failed in the fetcher step when it failed to fetch the required data from the beacon node API. This indicates a problem with the upstream beacon node.", + Long: "Reason `fetch_bn_error` indicates a duty failed in the fetcher step when it failed to fetch the required data from the beacon node API. This indicates a problem with the upstream beacon node.", } - reasonFetcherAggregatorNoAttData = reason{ - Code: "fetcher_aggregator_no_att_data", + reasonMissingAggregatorAttestation = reason{ + Code: "missing_aggregator_attestation", Short: "couldn't aggregate attestation due to failed attester duty", - Long: "Reason `fetcher_aggregator_no_att_data` indicates an attestation aggregation duty failed in the fetcher step since it couldn't fetch the prerequisite attestation data. This indicates the associated attestation duty failed to obtain a cluster agreed upon value.", + Long: "Reason `missing_aggregator_attestation` indicates an attestation aggregation duty failed in the fetcher step since it couldn't fetch the prerequisite attestation data. This indicates the associated attestation duty failed to obtain a cluster agreed upon value.", } - reasonFetcherAggregatorFewPrepares = reason{ - Code: "fetcher_aggregator_few_prepares", + reasonInsufficientAggregatorSelections = reason{ + Code: "insufficient_aggregator_selections", Short: "couldn't aggregate attestation due to insufficient partial beacon committee selections", - Long: "Reason `fetcher_aggregator_few_prepares` indicates an attestation aggregation duty failed in the fetcher step since it couldn't fetch the prerequisite aggregated beacon committee selections. This indicates the associated prepare aggregation duty failed due to insufficient partial beacon committee selections submitted by the cluster validator clients.", + Long: "Reason `insufficient_aggregator_selections` indicates an attestation aggregation duty failed in the fetcher step since it couldn't fetch the prerequisite aggregated beacon committee selections. This indicates the associated prepare aggregation duty failed due to insufficient partial beacon committee selections submitted by the cluster validator clients.", } - reasonFetcherAggregatorZeroPrepares = reason{ - Code: "fetcher_aggregator_zero_prepares", + reasonZeroAggregatorSelections = reason{ + Code: "zero_aggregator_prepares", Short: "couldn't aggregate attestation due to zero partial beacon committee selections", - Long: "Reason `fetcher_aggregator_zero_prepares` indicates an attestation aggregation duty failed in the fetcher step since it couldn't fetch the prerequisite aggregated beacon committee selections. This indicates the associated prepare aggregation duty failed due to no partial beacon committee selections submitted by the cluster validator clients.", + Long: "Reason `zero_aggregator_prepares` indicates an attestation aggregation duty failed in the fetcher step since it couldn't fetch the prerequisite aggregated beacon committee selections. This indicates the associated prepare aggregation duty failed due to no partial beacon committee selections submitted by the cluster validator clients.", } - reasonFetcherAggregatorFailedPrepare = reason{ - Code: "fetcher_aggregator_failed_prepare", + reasonFailedAggregatorSelection = reason{ + Code: "failed_aggregator_selection", Short: "couldn't aggregate attestation due to failed prepare aggregator duty", - Long: "Reason `fetcher_aggregator_failed_prepare` indicates an attestation aggregation duty failed in the fetcher step since it couldn't fetch the prerequisite aggregated beacon committee selections. This indicates the associated prepare aggregation duty failed.", + Long: "Reason `failed_aggregator_selection` indicates an attestation aggregation duty failed in the fetcher step since it couldn't fetch the prerequisite aggregated beacon committee selections. This indicates the associated prepare aggregation duty failed.", } - reasonFetcherAggregatorNoExternalPrepares = reason{ - Code: "fetcher_aggregator_no_external_prepares", + reasonNoAggregatorSelections = reason{ + Code: "no_aggregator_selections", Short: "couldn't aggregate attestation due to no partial beacon committee selections received from peers", - Long: "Reason `fetcher_aggregator_no_external_prepares` indicates an attestation aggregation duty failed in the fetcher step since it couldn't fetch the prerequisite aggregated beacon committee selections. This indicates the associated prepare aggregation duty failed due to no partial beacon committee selections received from peers.", + Long: "Reason `no_aggregator_selections` indicates an attestation aggregation duty failed in the fetcher step since it couldn't fetch the prerequisite aggregated beacon committee selections. This indicates the associated prepare aggregation duty failed due to no partial beacon committee selections received from peers.", } - reasonFetcherProposerFewRandaos = reason{ - Code: "fetcher_proposer_few_randaos", + reasonProposerInsufficientRandaos = reason{ + Code: "proposer_insufficient_randaos", Short: "couldn't propose block due to insufficient partial randao signatures", - Long: "Reason `fetcher_proposer_few_randaos` indicates a block proposer duty failed in the fetcher step since it couldn't fetch the prerequisite aggregated RANDAO. This indicates the associated randao duty failed due to insufficient partial randao signatures submitted by the cluster validator clients.", + Long: "Reason `proposer_insufficient_randaos` indicates a block proposer duty failed in the fetcher step since it couldn't fetch the prerequisite aggregated RANDAO. This indicates the associated randao duty failed due to insufficient partial randao signatures submitted by the cluster validator clients.", } - reasonFetcherProposerZeroRandaos = reason{ - Code: "fetcher_proposer_zero_randaos", + reasonProposerZeroRandaos = reason{ + Code: "proposer_zero_randaos", Short: "couldn't propose block due to zero partial randao signatures", - Long: "Reason `fetcher_proposer_zero_randaos` indicates a block proposer duty failed in the fetcher step since it couldn't fetch the prerequisite aggregated RANDAO. This indicates the associated randao duty failed due to no partial randao signatures submitted by the cluster validator clients.", + Long: "Reason `proposer_zero_randaos` indicates a block proposer duty failed in the fetcher step since it couldn't fetch the prerequisite aggregated RANDAO. This indicates the associated randao duty failed due to no partial randao signatures submitted by the cluster validator clients.", } - reasonFetcherProposerFailedRandao = reason{ - Code: "fetcher_proposer_failed_randao", + reasonFailedProposerRandao = reason{ + Code: "failed_proposer_randao", Short: "couldn't propose block due to failed randao duty", - Long: "msgFetcherProposerZeroRandaos indicates a block proposer duty failed in the fetcher step since it couldn't fetch the prerequisite aggregated RANDAO. This indicates the associated randao duty failed.", + Long: "Reason `failed_proposer_randao` indicates a block proposer duty failed in the fetcher step since it couldn't fetch the prerequisite aggregated RANDAO. This indicates the associated randao duty failed.", } - reasonFetcherProposerNoExternalRandaos = reason{ - Code: "fetcher_proposer_no_external_randaos", + reasonProposerNoExternalRandaos = reason{ + Code: "proposer_no_external_randaos", Short: "couldn't propose block due to no partial randao signatures received from peers", - Long: "Reason `fetcher_proposer_no_external_randaos` indicates a block proposer duty failed in the fetcher step since it couldn't fetch the prerequisite aggregated RANDAO. This indicates the associated randao duty failed due to no partial randao signatures received from peers.", + Long: "Reason `proposer_no_external_randaos` indicates a block proposer duty failed in the fetcher step since it couldn't fetch the prerequisite aggregated RANDAO. This indicates the associated randao duty failed due to no partial randao signatures received from peers.", } - reasonFetcherSyncContributionNoSyncMsg = reason{ - Code: "fetcher_sync_contribution_no_sync_msg", + reasonSyncContributionNoSyncMsg = reason{ + Code: "sync_contribution_no_sync_msg", Short: "couldn't fetch sync contribution due to failed sync message duty", - Long: "Reason `fetcher_sync_contribution_no_sync_msg` indicates a sync contribution duty failed in the fetcher step since it couldn't fetch the prerequisite sync message. This indicates the associated sync message duty failed to obtain a cluster agreed upon value.", + Long: "Reason `sync_contribution_no_sync_msg` indicates a sync contribution duty failed in the fetcher step since it couldn't fetch the prerequisite sync message. This indicates the associated sync message duty failed to obtain a cluster agreed upon value.", } - reasonFetcherSyncContributionFewPrepares = reason{ - Code: "fetcher_sync_contribution_few_prepares", + reasonSyncContributionFewPrepares = reason{ + Code: "sync_contribution_few_prepares", Short: "couldn't fetch sync contribution due to insufficient partial sync contribution selections", - Long: "Reason `fetcher_sync_contribution_few_prepares` indicates a sync contribution duty failed in the fetcher step since it couldn't fetch the prerequisite aggregated sync contribution selections. This indicates the associated prepare sync contribution duty failed due to insufficient partial sync contribution selections submitted by the cluster validator clients.", + Long: "Reason `sync_contribution_few_prepares` indicates a sync contribution duty failed in the fetcher step since it couldn't fetch the prerequisite aggregated sync contribution selections. This indicates the associated prepare sync contribution duty failed due to insufficient partial sync contribution selections submitted by the cluster validator clients.", } - reasonFetcherSyncContributionZeroPrepares = reason{ - Code: "fetcher_sync_contribution_zero_prepares", + reasonSyncContributionZeroPrepares = reason{ + Code: "sync_contribution_zero_prepares", Short: "couldn't fetch sync contribution due to zero partial sync contribution selections", - Long: "Reason `fetcher_sync_contribution_zero_prepares` indicates a sync contribution duty failed in the fetcher step since it couldn't fetch the prerequisite aggregated sync contribution selections. This indicates the associated prepare sync contribution duty failed due to no partial sync contribution selections submitted by the cluster validator clients.", + Long: "Reason `sync_contribution_zero_prepares` indicates a sync contribution duty failed in the fetcher step since it couldn't fetch the prerequisite aggregated sync contribution selections. This indicates the associated prepare sync contribution duty failed due to no partial sync contribution selections submitted by the cluster validator clients.", } - reasonFetcherSyncContributionFailedPrepare = reason{ - Code: "fetcher_sync_contribution_failed_prepare", + reasonSyncContributionFailedPrepare = reason{ + Code: "sync_contribution_failed_prepare", Short: "couldn't fetch sync contribution due to failed prepare sync contribution duty", - Long: "Reason `fetcher_sync_contribution_failed_prepare` indicates a sync contribution duty failed in the fetcher step since it couldn't fetch the prerequisite aggregated sync contribution selections. This indicates the associated prepare sync contribution duty failed.", + Long: "Reason `sync_contribution_failed_prepare` indicates a sync contribution duty failed in the fetcher step since it couldn't fetch the prerequisite aggregated sync contribution selections. This indicates the associated prepare sync contribution duty failed.", } - reasonFetcherSyncContributionNoExternalPrepares = reason{ - Code: "fetcher_sync_contribution_no_external_prepares", + reasonSyncContributionNoExternalPrepares = reason{ + Code: "sync_contribution_no_external_prepares", Short: "couldn't fetch sync contribution due to no partial sync contribution selections received from peers", - Long: "Reason `fetcher_sync_contribution_no_external_prepares` indicates a sync contribution duty failed in the fetcher step since it couldn't fetch the prerequisite aggregated sync contribution selections. This indicates the associated prepare sync contribution duty failed due to no partial sync contribution selections received from peers.", + Long: "Reason `sync_contribution_no_external_prepares` indicates a sync contribution duty failed in the fetcher step since it couldn't fetch the prerequisite aggregated sync contribution selections. This indicates the associated prepare sync contribution duty failed due to no partial sync contribution selections received from peers.", } - reasonFetcherError = reason{ - Code: "fetcher_error", - Short: "couldn't fetch due to unexpected error", - Long: "Reason `fetcher_error` indicates duty failed in fetcher step with some unexpected error. This indicates a problem in charon as it is unexpected.", - } - - reasonConsensus = reason{ - Code: "consensus", + reasonNoConsensus = reason{ + Code: "no_consensus", Short: "consensus algorithm didn't complete", - Long: "Reason `consensus` indicates a duty failed in consensus step. This could indicate that insufficient honest peers participated in consensus or p2p network connection problems.", - } - - reasonDutyDB = reason{ - Code: "duty_db", - Short: "bug: failed to store duty data in DutyDB", - Long: "Reason `duty_db` indicates a bug in the DutyDB database as it is unexpected.", + Long: "Reason `no_consensus` indicates a duty failed in consensus step. This could indicate that insufficient honest peers participated in consensus or p2p network connection problems.", } - reasonValidatorAPI = reason{ - Code: "validator_api", + reasonNoLocalVCSignature = reason{ + Code: "no_local_vc_signature", Short: "signed duty not submitted by local validator client", - Long: "Reason `validator_api` indicates that partial signature we never submitted by the local validator client. This could indicate that the local validator client is offline, or has connection problems with charon, or has some other problem. See validator client logs for more details.", + Long: "Reason `no_local_vc_signature` indicates that partial signature we never submitted by the local validator client. This could indicate that the local validator client is offline, or has connection problems with charon, or has some other problem. See validator client logs for more details.", } - reasonParSigDBInternal = reason{ - Code: "par_sig_db_internal", - Short: "partial signature database didn't trigger partial signature exchange, this is unexpected", - Long: "Reason `par_sig_db_internal` indicates a bug in the partial signature database as it is unexpected. Note this may happen due to expiry race.", - } - - reasonParSigExReceive = reason{ - Code: "par_sig_ex_receive", + reasonNoPeerSignatures = reason{ + Code: "no_peer_signatures", Short: "no partial signatures received from peers", - Long: "Reason `par_sig_ex_receive` indicates that no partial signature for the duty was received from any peer. This indicates all peers are offline or p2p network connection problems.", + Long: "Reason `no_peer_signatures` indicates that no partial signature for the duty was received from any peer. This indicates all peers are offline or p2p network connection problems.", } - reasonParSigDBInsufficient = reason{ - Code: "par_sig_db_insufficient", + reasonInsufficientPeerSignatures = reason{ + Code: "insufficient_peer_signatures", Short: "insufficient partial signatures received, minimum required threshold not reached", - Long: "Reason `par_sig_db_insufficient` indicates that insufficient partial signatures for the duty was received from peers. This indicates problems with peers or p2p network connection problems.", + Long: "Reason `insufficient_peer_signatures` indicates that insufficient partial signatures for the duty was received from peers. This indicates problems with peers or p2p network connection problems.", } reasonParSigDBInconsistentSync = reason{ @@ -157,39 +139,57 @@ var ( Long: "Reason `par_sig_db_inconsistent_sync` indicates that partial signed data for the sync committee duty were inconsistent. This is known limitation in this version of charon.", } - reasonParSigDBInconsistent = reason{ - Code: "par_sig_db_inconsistent", + reasonBroadcastBNError = reason{ + Code: "broadcast_bn_error", + Short: "failed to broadcast duty to beacon node", + Long: "Reason `broadcast_bn_error` indicates that beacon node returned an error while submitting aggregated duty signature to beacon node.", + } + + reasonNotIncludedOnChain = reason{ + Code: "not_included_onchain", + Short: "duty not included on-chain", + Long: "Reason `not_included_onchain` indicates that even though charon broadcasted the duty successfully, it wasn't included in the beacon chain. This is expected for up to 20% of attestations. It may however indicate problematic charon broadcast delays or beacon node network problems.", + } + + reasonBugFetchError = reason{ + Code: "bug_fetch_error", + Short: "bug: couldn't fetch due to unexpected error", + Long: "Reason `bug_fetch_error` indicates duty failed in fetcher step with some unexpected error. This indicates a problem in charon as it is unexpected.", + } + + reasonBugParSigDBInconsistent = reason{ + Code: "bug_par_sig_db_inconsistent", Short: "bug: inconsistent partial signatures received", - Long: "Reason `par_sig_db_inconsistent` indicates that partial signed data for the duty were inconsistent. This indicates a bug in charon as it is unexpected (for non-sync-committee-duties).", + Long: "Reason `bug_par_sig_db_inconsistent` indicates that partial signed data for the duty were inconsistent. This indicates a bug in charon as it is unexpected (for non-sync-committee-duties).", } - reasonParSigDBExternal = reason{ - Code: "par_sig_db_external", + reasonBugParSigDBExternal = reason{ + Code: "bug_par_sig_db_external", Short: "bug: failed to store external partial signatures in parsigdb", - Long: "Reason `par_sig_db_external` indicates a bug in the partial signature database as it is unexpected.", + Long: "Reason `bug_par_sig_db_external` indicates a bug in the partial signature database as it is unexpected.", } - reasonSigAgg = reason{ - Code: "sig_agg", + reasonBugSigAgg = reason{ + Code: "bug_sig_agg", Short: "bug: threshold aggregation of partial signatures failed due to inconsistent signed data", - Long: "Reason `sig_agg` indicates that BLS threshold aggregation of sufficient partial signatures failed. This indicates inconsistent signed data. This indicates a bug in charon as it is unexpected.", + Long: "Reason `bug_sig_agg` indicates that BLS threshold aggregation of sufficient partial signatures failed. This indicates inconsistent signed data. This indicates a bug in charon as it is unexpected.", } - reasonAggSigDB = reason{ - Code: "agg_sig_db", + reasonBugAggregationError = reason{ + Code: "bug_aggregation_error", Short: "bug: failed to store aggregated signature in aggsigdb", - Long: "Reason `agg_sig_db` indicates a bug in the aggregated signature database as it is unexpected.", + Long: "Reason `bug_aggregation_error` indicates a bug in the aggregated signature database as it is unexpected.", } - reasonBcast = reason{ - Code: "bcast", - Short: "failed to broadcast duty to beacon node", - Long: "Reason `bcast` indicates that beacon node returned an error while submitting aggregated duty signature to beacon node.", + reasonBugDutyDBError = reason{ + Code: "bug_duty_db_error", + Short: "bug: failed to store duty data in DutyDB", + Long: "Reason `bug_duty_db_error` indicates a bug in the DutyDB database as it is unexpected.", } - reasonChainIncl = reason{ - Code: "chain_inclusion", - Short: "duty not included on-chain", - Long: "Reason `chain_inclusion` indicates that even though charon broadcasted the duty successfully, it wasn't included in the beacon chain. This is expected for up to 20% of attestations. It may however indicate problematic charon broadcast delays or beacon node network problems.", + reasonBugParSigDBInternal = reason{ + Code: "bug_par_sig_db_internal", + Short: "bug: partial signature database didn't trigger partial signature exchange, this is unexpected", + Long: "Reason `bug_par_sig_db_internal` indicates a bug in the partial signature database as it is unexpected. Note this may happen due to expiry race.", } ) diff --git a/core/tracker/tracker.go b/core/tracker/tracker.go index e31739e39..5e26102d2 100644 --- a/core/tracker/tracker.go +++ b/core/tracker/tracker.go @@ -233,51 +233,51 @@ func analyseDutyFailed(duty core.Duty, allEvents map[core.Duty][]event, msgRootC return analyseFetcherFailed(duty, allEvents, failedErr) case consensus: if failedErr != nil { - reason = reasonConsensus + reason = reasonNoConsensus } case dutyDB: if failedErr != nil { - reason = reasonDutyDB + reason = reasonBugDutyDBError } else { failedStep = validatorAPI - reason = reasonValidatorAPI + reason = reasonNoLocalVCSignature } case parSigDBInternal: - reason = reasonParSigDBInternal + reason = reasonBugParSigDBInternal case parSigEx: if failedErr == nil { - reason = reasonParSigExReceive + reason = reasonNoPeerSignatures } case parSigDBExternal: if failedErr != nil { - return true, parSigDBExternal, reasonParSigDBExternal, failedErr + return true, parSigDBExternal, reasonBugParSigDBExternal, failedErr } if msgRootConsistent { - reason = reasonParSigDBInsufficient + reason = reasonInsufficientPeerSignatures } else { - reason = reasonParSigDBInconsistent + reason = reasonBugParSigDBInconsistent if expectInconsistentParSigs(duty.Type) { reason = reasonParSigDBInconsistentSync } } case sigAgg: if failedErr != nil { - reason = reasonSigAgg + reason = reasonBugSigAgg } case aggSigDB: - reason = reasonAggSigDB + reason = reasonBugAggregationError case bcast: if failedErr == nil { failedErr = errors.New("bug: missing chain inclusion event") } else { - reason = reasonBcast + reason = reasonBroadcastBNError } case chainInclusion: if failedErr == nil { failedErr = errors.New("bug: missing chain inclusion error") } else { - reason = reasonChainIncl + reason = reasonNotIncludedOnChain } case zero: failedErr = errors.New("no events for duty") // This should never happen. @@ -291,13 +291,13 @@ func analyseDutyFailed(duty core.Duty, allEvents map[core.Duty][]event, msgRootC // analyseFetcherFailed returns whether the duty that got stuck in fetcher actually failed // and the reason which might actually be due a pre-requisite duty that failed. func analyseFetcherFailed(duty core.Duty, allEvents map[core.Duty][]event, fetchErr error) (bool, step, reason, error) { - reason := reasonFetcherError + reason := reasonBugFetchError // Check for beacon api errors. var eth2Error eth2api.Error if errors.As(fetchErr, ð2Error) { - reason = reasonFetcherBN + reason = reasonFetchBNError } else if !errors.Is(fetchErr, context.Canceled) && !errors.Is(fetchErr, context.DeadlineExceeded) { - reason = reasonFetcherError + reason = reasonBugFetchError } // Proposer duties depend on randao duty, so check if that was why it failed. @@ -321,7 +321,7 @@ func analyseFetcherFailed(duty core.Duty, allEvents map[core.Duty][]event, fetch // analyseFetcherFailed returns the reason behind why proposer duty failed which might actually // be due to randao duty failed. func analyseFetcherFailedProposer(duty core.Duty, allEvents map[core.Duty][]event, fetchErr error) (bool, step, reason, error) { - reason := reasonFetcherError + reason := reasonBugFetchError // Proposer duties will fail if core.DutyRandao fails. // Ignoring error as it will be handled in DutyRandao analysis. @@ -329,13 +329,13 @@ func analyseFetcherFailedProposer(duty core.Duty, allEvents map[core.Duty][]even if randaoFailed { switch randaoStep { case parSigEx: - reason = reasonFetcherProposerNoExternalRandaos + reason = reasonProposerNoExternalRandaos case parSigDBExternal: - reason = reasonFetcherProposerFewRandaos + reason = reasonProposerInsufficientRandaos case zero: - reason = reasonFetcherProposerZeroRandaos + reason = reasonProposerZeroRandaos default: - reason = reasonFetcherProposerFailedRandao + reason = reasonFailedProposerRandao } } @@ -345,7 +345,7 @@ func analyseFetcherFailedProposer(duty core.Duty, allEvents map[core.Duty][]even // analyseFetcherFailedAggregator returns the reason behind why aggregator duty failed which might actually // be due to prepare aggregator duty or attester duty failed. func analyseFetcherFailedAggregator(duty core.Duty, allEvents map[core.Duty][]event, fetchErr error) (bool, step, reason, error) { - failedReason := reasonFetcherError + failedReason := reasonBugFetchError // No aggregators present for this slot. if fetchErr == nil { @@ -358,13 +358,13 @@ func analyseFetcherFailedAggregator(duty core.Duty, allEvents map[core.Duty][]ev if prepAggFailed { switch prepAggStep { case parSigEx: - failedReason = reasonFetcherAggregatorNoExternalPrepares + failedReason = reasonNoAggregatorSelections case parSigDBExternal: - failedReason = reasonFetcherAggregatorFewPrepares + failedReason = reasonInsufficientAggregatorSelections case zero: - failedReason = reasonFetcherAggregatorZeroPrepares + failedReason = reasonZeroAggregatorSelections default: - failedReason = reasonFetcherAggregatorFailedPrepare + failedReason = reasonFailedAggregatorSelection } return true, fetcher, failedReason, fetchErr @@ -374,7 +374,7 @@ func analyseFetcherFailedAggregator(duty core.Duty, allEvents map[core.Duty][]ev // Ignoring error as it will be handled in DutyAttester analysis. attFailed, attStep, _ := dutyFailedStep(allEvents[core.NewAttesterDuty(duty.Slot)]) if attFailed && attStep <= dutyDB { - failedReason = reasonFetcherAggregatorNoAttData + failedReason = reasonMissingAggregatorAttestation } return true, fetcher, failedReason, fetchErr @@ -383,7 +383,7 @@ func analyseFetcherFailedAggregator(duty core.Duty, allEvents map[core.Duty][]ev // analyseFetcherFailedSyncContribution returns the reason behind why sync contribution duty failed which might actually // be due to prepare sync contribution duty or sync message duty failed. func analyseFetcherFailedSyncContribution(duty core.Duty, allEvents map[core.Duty][]event, fetchErr error) (bool, step, reason, error) { - failReason := reasonFetcherError + failReason := reasonBugFetchError // No sync committee aggregators present for this slot. if fetchErr == nil { @@ -396,13 +396,13 @@ func analyseFetcherFailedSyncContribution(duty core.Duty, allEvents map[core.Dut if prepSyncConFailed { switch prepSyncConStep { case parSigEx: - failReason = reasonFetcherSyncContributionNoExternalPrepares + failReason = reasonSyncContributionNoExternalPrepares case parSigDBExternal: - failReason = reasonFetcherSyncContributionFewPrepares + failReason = reasonSyncContributionFewPrepares case zero: - failReason = reasonFetcherSyncContributionZeroPrepares + failReason = reasonSyncContributionZeroPrepares default: - failReason = reasonFetcherSyncContributionFailedPrepare + failReason = reasonSyncContributionFailedPrepare } return true, fetcher, failReason, fetchErr @@ -412,7 +412,7 @@ func analyseFetcherFailedSyncContribution(duty core.Duty, allEvents map[core.Dut // Ignoring error as it will be handled in DutySyncMessage analysis. syncMsgFailed, syncMsgStep, _ := dutyFailedStep(allEvents[core.NewSyncMessageDuty(duty.Slot)]) if syncMsgFailed && syncMsgStep <= aggSigDB { - failReason = reasonFetcherSyncContributionNoSyncMsg + failReason = reasonSyncContributionNoSyncMsg } return true, fetcher, failReason, fetchErr @@ -511,7 +511,7 @@ func newUnsupportedIgnorer() func(ctx context.Context, duty core.Duty, failed bo return false } - if !aggregationSupported && duty.Type == core.DutyAggregator && step == fetcher && reason == reasonFetcherAggregatorZeroPrepares { + if !aggregationSupported && duty.Type == core.DutyAggregator && step == fetcher && reason == reasonZeroAggregatorSelections { if !loggedNoAggregator { log.Warn(ctx, "Ignoring attestation aggregation failures since VCs do not seem to support beacon committee selection aggregation", nil) } @@ -520,7 +520,7 @@ func newUnsupportedIgnorer() func(ctx context.Context, duty core.Duty, failed bo return true } - if !contributionSupported && duty.Type == core.DutySyncContribution && step == fetcher && reason == reasonFetcherSyncContributionZeroPrepares { + if !contributionSupported && duty.Type == core.DutySyncContribution && step == fetcher && reason == reasonSyncContributionZeroPrepares { if !loggedNoContribution { log.Warn(ctx, "Ignoring sync contribution failures since VCs do not seem to support sync committee selection aggregation", nil) } diff --git a/core/tracker/tracker_internal_test.go b/core/tracker/tracker_internal_test.go index f6fb9121c..fdf112e3a 100644 --- a/core/tracker/tracker_internal_test.go +++ b/core/tracker/tracker_internal_test.go @@ -41,7 +41,7 @@ func TestTrackerFailedDuty(t *testing.T) { require.Equal(t, testData[0].duty, failedDuty) require.True(t, isFailed) require.Equal(t, consensus, step) - require.Equal(t, reason, reasonConsensus) + require.Equal(t, reason, reasonNoConsensus) require.ErrorIs(t, err, consensusErr) count++ @@ -145,7 +145,7 @@ func TestAnalyseDutyFailed(t *testing.T) { require.ErrorAs(t, err, &fetcherErr) require.True(t, failed) require.Equal(t, step, fetcher) - require.Equal(t, reason, reasonFetcherError) + require.Equal(t, reason, reasonBugFetchError) }) t.Run("Failed at consensus", func(t *testing.T) { @@ -160,7 +160,7 @@ func TestAnalyseDutyFailed(t *testing.T) { require.ErrorAs(t, err, &consensusErr) require.True(t, failed) require.Equal(t, step, consensus) - require.Equal(t, reason, reasonConsensus) + require.Equal(t, reason, reasonNoConsensus) }) // Failed at validatorAPI @@ -174,7 +174,7 @@ func TestAnalyseDutyFailed(t *testing.T) { require.NoError(t, err) require.True(t, failed) require.Equal(t, step, validatorAPI) - require.Equal(t, reason, reasonValidatorAPI) + require.Equal(t, reason, reasonNoLocalVCSignature) }) // Failed at parsigDBInternal t.Run("Failed at parsigDBInternal", func(t *testing.T) { @@ -189,7 +189,7 @@ func TestAnalyseDutyFailed(t *testing.T) { require.ErrorAs(t, err, &parSigDBInternalErr) require.True(t, failed) require.Equal(t, step, parSigDBInternal) - require.Equal(t, reason, reasonParSigDBInternal) + require.Equal(t, reason, reasonBugParSigDBInternal) }) // Failed at parsigEx @@ -203,7 +203,7 @@ func TestAnalyseDutyFailed(t *testing.T) { require.NoError(t, err) require.True(t, failed) require.Equal(t, step, parSigEx) - require.Equal(t, reason, reasonParSigExReceive) + require.Equal(t, reason, reasonNoPeerSignatures) }) // Failed at parsigDBInternal @@ -219,7 +219,7 @@ func TestAnalyseDutyFailed(t *testing.T) { require.ErrorAs(t, err, &parSigDBExternalErr) require.True(t, failed) require.Equal(t, step, parSigDBExternal) - require.Equal(t, reason, reasonParSigDBExternal) + require.Equal(t, reason, reasonBugParSigDBExternal) }) t.Run("Failed at parsigDBThreshold", func(t *testing.T) { @@ -232,13 +232,13 @@ func TestAnalyseDutyFailed(t *testing.T) { require.NoError(t, err) require.True(t, failed) require.Equal(t, step, parSigDBExternal) - require.Equal(t, reason, reasonParSigDBInsufficient) + require.Equal(t, reason, reasonInsufficientPeerSignatures) failed, step, reason, err = analyseDutyFailed(attDuty, events, false) require.NoError(t, err) require.True(t, failed) require.Equal(t, step, parSigDBExternal) - require.Equal(t, reason, reasonParSigDBInconsistent) + require.Equal(t, reason, reasonBugParSigDBInconsistent) events[syncMsgDuty] = events[attDuty] failed, step, reason, err = analyseDutyFailed(syncMsgDuty, events, false) @@ -260,7 +260,7 @@ func TestAnalyseDutyFailed(t *testing.T) { require.ErrorIs(t, err, bcastErr) require.True(t, failed) require.Equal(t, step, bcast) - require.Equal(t, reason, reasonBcast) + require.Equal(t, reason, reasonBroadcastBNError) }) t.Run("Failed at chain inclusion", func(t *testing.T) { @@ -275,7 +275,7 @@ func TestAnalyseDutyFailed(t *testing.T) { require.ErrorIs(t, err, inclErr) require.True(t, failed) require.Equal(t, step, chainInclusion) - require.Equal(t, reason, reasonChainIncl) + require.Equal(t, reason, reasonNotIncludedOnChain) }) t.Run("FailedAtFetcherAsRandaoFailed", func(t *testing.T) { @@ -309,7 +309,7 @@ func TestAnalyseDutyFailed(t *testing.T) { require.ErrorAs(t, err, &expectedErr) require.True(t, failed) require.Equal(t, step, fetcher) - require.Equal(t, reason, reasonFetcherProposerNoExternalRandaos) + require.Equal(t, reason, reasonProposerNoExternalRandaos) // Randao failed at parSigDBThreshold events[randaoDuty] = append(events[randaoDuty], event{ @@ -321,7 +321,7 @@ func TestAnalyseDutyFailed(t *testing.T) { require.ErrorAs(t, err, &expectedErr) require.True(t, failed) require.Equal(t, step, fetcher) - require.Equal(t, reason, reasonFetcherProposerFewRandaos) + require.Equal(t, reason, reasonProposerInsufficientRandaos) // No Randaos events[randaoDuty] = nil @@ -330,7 +330,7 @@ func TestAnalyseDutyFailed(t *testing.T) { require.ErrorAs(t, err, &expectedErr) require.True(t, failed) require.Equal(t, step, fetcher) - require.Equal(t, reason, reasonFetcherProposerZeroRandaos) + require.Equal(t, reason, reasonProposerZeroRandaos) }) t.Run("Attester Duty Success", func(t *testing.T) { @@ -768,7 +768,7 @@ func TestAnalyseFetcherFailed(t *testing.T) { }, "beacon api error"), }}, }, - reason: reasonFetcherBN, + reason: reasonFetchBNError, failed: true, err: errors.Wrap(eth2api.Error{ Method: http.MethodGet, @@ -787,7 +787,7 @@ func TestAnalyseFetcherFailed(t *testing.T) { stepErr: context.Canceled, }}, }, - reason: reasonFetcherAggregatorZeroPrepares, + reason: reasonZeroAggregatorSelections, failed: true, err: context.Canceled, }, @@ -805,7 +805,7 @@ func TestAnalyseFetcherFailed(t *testing.T) { step: parSigEx, }}, }, - reason: reasonFetcherAggregatorNoExternalPrepares, + reason: reasonNoAggregatorSelections, failed: true, err: context.Canceled, }, @@ -823,7 +823,7 @@ func TestAnalyseFetcherFailed(t *testing.T) { step: parSigDBExternal, }}, }, - reason: reasonFetcherAggregatorFewPrepares, + reason: reasonInsufficientAggregatorSelections, failed: true, err: context.Canceled, }, @@ -841,7 +841,7 @@ func TestAnalyseFetcherFailed(t *testing.T) { step: sigAgg, }}, }, - reason: reasonFetcherAggregatorFailedPrepare, + reason: reasonFailedAggregatorSelection, failed: true, err: context.Canceled, }, @@ -864,7 +864,7 @@ func TestAnalyseFetcherFailed(t *testing.T) { stepErr: errors.New("some error"), }}, }, - reason: reasonFetcherAggregatorNoAttData, + reason: reasonMissingAggregatorAttestation, failed: true, err: context.Canceled, }, @@ -898,7 +898,7 @@ func TestAnalyseFetcherFailed(t *testing.T) { stepErr: context.Canceled, }}, }, - reason: reasonFetcherSyncContributionZeroPrepares, + reason: reasonSyncContributionZeroPrepares, failed: true, err: context.Canceled, }, @@ -916,7 +916,7 @@ func TestAnalyseFetcherFailed(t *testing.T) { step: parSigEx, }}, }, - reason: reasonFetcherSyncContributionNoExternalPrepares, + reason: reasonSyncContributionNoExternalPrepares, failed: true, err: context.Canceled, }, @@ -934,7 +934,7 @@ func TestAnalyseFetcherFailed(t *testing.T) { step: parSigDBExternal, }}, }, - reason: reasonFetcherSyncContributionFewPrepares, + reason: reasonSyncContributionFewPrepares, failed: true, err: context.Canceled, }, @@ -952,7 +952,7 @@ func TestAnalyseFetcherFailed(t *testing.T) { step: sigAgg, }}, }, - reason: reasonFetcherSyncContributionFailedPrepare, + reason: reasonSyncContributionFailedPrepare, failed: true, err: context.Canceled, }, @@ -974,7 +974,7 @@ func TestAnalyseFetcherFailed(t *testing.T) { step: parSigEx, }}, }, - reason: reasonFetcherSyncContributionNoSyncMsg, + reason: reasonSyncContributionNoSyncMsg, failed: true, err: context.Canceled, }, @@ -1010,7 +1010,7 @@ func TestAnalyseFetcherFailed(t *testing.T) { }, }, }, - reason: reasonFetcherError, + reason: reasonBugFetchError, failed: true, err: errors.New("unexpected error"), }, @@ -1241,7 +1241,7 @@ func TestIgnoreUnsupported(t *testing.T) { duty: core.NewAttesterDuty(123), failed: testutil.RandomBool(), step: randomStep(), - reason: reasonAggSigDB, + reason: reasonBugAggregationError, result: false, }, { @@ -1249,7 +1249,7 @@ func TestIgnoreUnsupported(t *testing.T) { duty: core.NewAggregatorDuty(123), failed: true, step: fetcher, - reason: reasonFetcherAggregatorZeroPrepares, + reason: reasonZeroAggregatorSelections, result: true, }, { @@ -1257,7 +1257,7 @@ func TestIgnoreUnsupported(t *testing.T) { duty: core.NewAggregatorDuty(123), failed: false, step: fetcher, - reason: reasonFetcherAggregatorZeroPrepares, + reason: reasonZeroAggregatorSelections, result: false, }, { @@ -1265,7 +1265,7 @@ func TestIgnoreUnsupported(t *testing.T) { duty: core.NewAggregatorDuty(123), failed: true, step: fetcher, - reason: reasonFetcherAggregatorZeroPrepares, + reason: reasonZeroAggregatorSelections, result: false, }, { @@ -1273,7 +1273,7 @@ func TestIgnoreUnsupported(t *testing.T) { duty: core.NewSyncContributionDuty(123), failed: true, step: fetcher, - reason: reasonFetcherSyncContributionZeroPrepares, + reason: reasonSyncContributionZeroPrepares, result: true, }, { @@ -1281,7 +1281,7 @@ func TestIgnoreUnsupported(t *testing.T) { duty: core.NewSyncContributionDuty(123), failed: false, step: fetcher, - reason: reasonFetcherSyncContributionZeroPrepares, + reason: reasonSyncContributionZeroPrepares, result: false, }, { @@ -1289,7 +1289,7 @@ func TestIgnoreUnsupported(t *testing.T) { duty: core.NewSyncContributionDuty(123), failed: true, step: fetcher, - reason: reasonFetcherSyncContributionZeroPrepares, + reason: reasonSyncContributionZeroPrepares, result: false, }, } diff --git a/docs/reasons.md b/docs/reasons.md index 23471b1e0..d5728afe8 100644 --- a/docs/reasons.md +++ b/docs/reasons.md @@ -9,122 +9,122 @@ By understanding these failure reasons, operators can better monitor, troublesho maintain system performance. -### Failure Reason: `agg_sig_db` +### Failure Reason: `broadcast_bn_error` + - *Summary*: failed to broadcast duty to beacon node + - *Details*: Reason `broadcast_bn_error` indicates that beacon node returned an error while submitting aggregated duty signature to beacon node. + +### Failure Reason: `bug_aggregation_error` - *Summary*: bug: failed to store aggregated signature in aggsigdb - - *Details*: Reason `agg_sig_db` indicates a bug in the aggregated signature database as it is unexpected. + - *Details*: Reason `bug_aggregation_error` indicates a bug in the aggregated signature database as it is unexpected. -### Failure Reason: `bcast` - - *Summary*: failed to broadcast duty to beacon node - - *Details*: Reason `bcast` indicates that beacon node returned an error while submitting aggregated duty signature to beacon node. +### Failure Reason: `bug_duty_db_error` + - *Summary*: bug: failed to store duty data in DutyDB + - *Details*: Reason `bug_duty_db_error` indicates a bug in the DutyDB database as it is unexpected. -### Failure Reason: `chain_inclusion` - - *Summary*: duty not included on-chain - - *Details*: Reason `chain_inclusion` indicates that even though charon broadcasted the duty successfully, it wasn`t included in the beacon chain. This is expected for up to 20% of attestations. It may however indicate problematic charon broadcast delays or beacon node network problems. +### Failure Reason: `bug_fetch_error` + - *Summary*: bug: couldn`t fetch due to unexpected error + - *Details*: Reason `bug_fetch_error` indicates duty failed in fetcher step with some unexpected error. This indicates a problem in charon as it is unexpected. -### Failure Reason: `consensus` - - *Summary*: consensus algorithm didn`t complete - - *Details*: Reason `consensus` indicates a duty failed in consensus step. This could indicate that insufficient honest peers participated in consensus or p2p network connection problems. +### Failure Reason: `bug_par_sig_db_external` + - *Summary*: bug: failed to store external partial signatures in parsigdb + - *Details*: Reason `bug_par_sig_db_external` indicates a bug in the partial signature database as it is unexpected. -### Failure Reason: `duty_db` - - *Summary*: bug: failed to store duty data in DutyDB - - *Details*: Reason `duty_db` indicates a bug in the DutyDB database as it is unexpected. +### Failure Reason: `bug_par_sig_db_inconsistent` + - *Summary*: bug: inconsistent partial signatures received + - *Details*: Reason `bug_par_sig_db_inconsistent` indicates that partial signed data for the duty were inconsistent. This indicates a bug in charon as it is unexpected (for non-sync-committee-duties). + +### Failure Reason: `bug_par_sig_db_internal` + - *Summary*: bug: partial signature database didn`t trigger partial signature exchange, this is unexpected + - *Details*: Reason `bug_par_sig_db_internal` indicates a bug in the partial signature database as it is unexpected. Note this may happen due to expiry race. + +### Failure Reason: `bug_sig_agg` + - *Summary*: bug: threshold aggregation of partial signatures failed due to inconsistent signed data + - *Details*: Reason `bug_sig_agg` indicates that BLS threshold aggregation of sufficient partial signatures failed. This indicates inconsistent signed data. This indicates a bug in charon as it is unexpected. -### Failure Reason: `fetcher_aggregator_failed_prepare` +### Failure Reason: `failed_aggregator_selection` - *Summary*: couldn`t aggregate attestation due to failed prepare aggregator duty - - *Details*: Reason `fetcher_aggregator_failed_prepare` indicates an attestation aggregation duty failed in the fetcher step since it couldn`t fetch the prerequisite aggregated beacon committee selections. This indicates the associated prepare aggregation duty failed. + - *Details*: Reason `failed_aggregator_selection` indicates an attestation aggregation duty failed in the fetcher step since it couldn`t fetch the prerequisite aggregated beacon committee selections. This indicates the associated prepare aggregation duty failed. -### Failure Reason: `fetcher_aggregator_few_prepares` +### Failure Reason: `failed_proposer_randao` + - *Summary*: couldn`t propose block due to failed randao duty + - *Details*: Reason `failed_proposer_randao` indicates a block proposer duty failed in the fetcher step since it couldn`t fetch the prerequisite aggregated RANDAO. This indicates the associated randao duty failed. + +### Failure Reason: `fetch_bn_error` + - *Summary*: couldn`t fetch duty data from the beacon node + - *Details*: Reason `fetch_bn_error` indicates a duty failed in the fetcher step when it failed to fetch the required data from the beacon node API. This indicates a problem with the upstream beacon node. + +### Failure Reason: `insufficient_aggregator_selections` - *Summary*: couldn`t aggregate attestation due to insufficient partial beacon committee selections - - *Details*: Reason `fetcher_aggregator_few_prepares` indicates an attestation aggregation duty failed in the fetcher step since it couldn`t fetch the prerequisite aggregated beacon committee selections. This indicates the associated prepare aggregation duty failed due to insufficient partial beacon committee selections submitted by the cluster validator clients. + - *Details*: Reason `insufficient_aggregator_selections` indicates an attestation aggregation duty failed in the fetcher step since it couldn`t fetch the prerequisite aggregated beacon committee selections. This indicates the associated prepare aggregation duty failed due to insufficient partial beacon committee selections submitted by the cluster validator clients. + +### Failure Reason: `insufficient_peer_signatures` + - *Summary*: insufficient partial signatures received, minimum required threshold not reached + - *Details*: Reason `insufficient_peer_signatures` indicates that insufficient partial signatures for the duty was received from peers. This indicates problems with peers or p2p network connection problems. -### Failure Reason: `fetcher_aggregator_no_att_data` +### Failure Reason: `missing_aggregator_attestation` - *Summary*: couldn`t aggregate attestation due to failed attester duty - - *Details*: Reason `fetcher_aggregator_no_att_data` indicates an attestation aggregation duty failed in the fetcher step since it couldn`t fetch the prerequisite attestation data. This indicates the associated attestation duty failed to obtain a cluster agreed upon value. + - *Details*: Reason `missing_aggregator_attestation` indicates an attestation aggregation duty failed in the fetcher step since it couldn`t fetch the prerequisite attestation data. This indicates the associated attestation duty failed to obtain a cluster agreed upon value. -### Failure Reason: `fetcher_aggregator_no_external_prepares` +### Failure Reason: `no_aggregator_selections` - *Summary*: couldn`t aggregate attestation due to no partial beacon committee selections received from peers - - *Details*: Reason `fetcher_aggregator_no_external_prepares` indicates an attestation aggregation duty failed in the fetcher step since it couldn`t fetch the prerequisite aggregated beacon committee selections. This indicates the associated prepare aggregation duty failed due to no partial beacon committee selections received from peers. + - *Details*: Reason `no_aggregator_selections` indicates an attestation aggregation duty failed in the fetcher step since it couldn`t fetch the prerequisite aggregated beacon committee selections. This indicates the associated prepare aggregation duty failed due to no partial beacon committee selections received from peers. -### Failure Reason: `fetcher_aggregator_zero_prepares` - - *Summary*: couldn`t aggregate attestation due to zero partial beacon committee selections - - *Details*: Reason `fetcher_aggregator_zero_prepares` indicates an attestation aggregation duty failed in the fetcher step since it couldn`t fetch the prerequisite aggregated beacon committee selections. This indicates the associated prepare aggregation duty failed due to no partial beacon committee selections submitted by the cluster validator clients. +### Failure Reason: `no_consensus` + - *Summary*: consensus algorithm didn`t complete + - *Details*: Reason `no_consensus` indicates a duty failed in consensus step. This could indicate that insufficient honest peers participated in consensus or p2p network connection problems. -### Failure Reason: `fetcher_bn` - - *Summary*: couldn`t fetch duty data from the beacon node - - *Details*: Reason `fetcher_bn` indicates a duty failed in the fetcher step when it failed to fetch the required data from the beacon node API. This indicates a problem with the upstream beacon node. +### Failure Reason: `no_local_vc_signature` + - *Summary*: signed duty not submitted by local validator client + - *Details*: Reason `no_local_vc_signature` indicates that partial signature we never submitted by the local validator client. This could indicate that the local validator client is offline, or has connection problems with charon, or has some other problem. See validator client logs for more details. + +### Failure Reason: `no_peer_signatures` + - *Summary*: no partial signatures received from peers + - *Details*: Reason `no_peer_signatures` indicates that no partial signature for the duty was received from any peer. This indicates all peers are offline or p2p network connection problems. -### Failure Reason: `fetcher_error` - - *Summary*: couldn`t fetch due to unexpected error - - *Details*: Reason `fetcher_error` indicates duty failed in fetcher step with some unexpected error. This indicates a problem in charon as it is unexpected. +### Failure Reason: `not_included_onchain` + - *Summary*: duty not included on-chain + - *Details*: Reason `not_included_onchain` indicates that even though charon broadcasted the duty successfully, it wasn`t included in the beacon chain. This is expected for up to 20% of attestations. It may however indicate problematic charon broadcast delays or beacon node network problems. -### Failure Reason: `fetcher_proposer_failed_randao` - - *Summary*: couldn`t propose block due to failed randao duty - - *Details*: msgFetcherProposerZeroRandaos indicates a block proposer duty failed in the fetcher step since it couldn`t fetch the prerequisite aggregated RANDAO. This indicates the associated randao duty failed. +### Failure Reason: `par_sig_db_inconsistent_sync` + - *Summary*: known limitation: inconsistent sync committee signatures received + - *Details*: Reason `par_sig_db_inconsistent_sync` indicates that partial signed data for the sync committee duty were inconsistent. This is known limitation in this version of charon. -### Failure Reason: `fetcher_proposer_few_randaos` +### Failure Reason: `proposer_insufficient_randaos` - *Summary*: couldn`t propose block due to insufficient partial randao signatures - - *Details*: Reason `fetcher_proposer_few_randaos` indicates a block proposer duty failed in the fetcher step since it couldn`t fetch the prerequisite aggregated RANDAO. This indicates the associated randao duty failed due to insufficient partial randao signatures submitted by the cluster validator clients. + - *Details*: Reason `proposer_insufficient_randaos` indicates a block proposer duty failed in the fetcher step since it couldn`t fetch the prerequisite aggregated RANDAO. This indicates the associated randao duty failed due to insufficient partial randao signatures submitted by the cluster validator clients. -### Failure Reason: `fetcher_proposer_no_external_randaos` +### Failure Reason: `proposer_no_external_randaos` - *Summary*: couldn`t propose block due to no partial randao signatures received from peers - - *Details*: Reason `fetcher_proposer_no_external_randaos` indicates a block proposer duty failed in the fetcher step since it couldn`t fetch the prerequisite aggregated RANDAO. This indicates the associated randao duty failed due to no partial randao signatures received from peers. + - *Details*: Reason `proposer_no_external_randaos` indicates a block proposer duty failed in the fetcher step since it couldn`t fetch the prerequisite aggregated RANDAO. This indicates the associated randao duty failed due to no partial randao signatures received from peers. -### Failure Reason: `fetcher_proposer_zero_randaos` +### Failure Reason: `proposer_zero_randaos` - *Summary*: couldn`t propose block due to zero partial randao signatures - - *Details*: Reason `fetcher_proposer_zero_randaos` indicates a block proposer duty failed in the fetcher step since it couldn`t fetch the prerequisite aggregated RANDAO. This indicates the associated randao duty failed due to no partial randao signatures submitted by the cluster validator clients. + - *Details*: Reason `proposer_zero_randaos` indicates a block proposer duty failed in the fetcher step since it couldn`t fetch the prerequisite aggregated RANDAO. This indicates the associated randao duty failed due to no partial randao signatures submitted by the cluster validator clients. -### Failure Reason: `fetcher_sync_contribution_failed_prepare` +### Failure Reason: `sync_contribution_failed_prepare` - *Summary*: couldn`t fetch sync contribution due to failed prepare sync contribution duty - - *Details*: Reason `fetcher_sync_contribution_failed_prepare` indicates a sync contribution duty failed in the fetcher step since it couldn`t fetch the prerequisite aggregated sync contribution selections. This indicates the associated prepare sync contribution duty failed. + - *Details*: Reason `sync_contribution_failed_prepare` indicates a sync contribution duty failed in the fetcher step since it couldn`t fetch the prerequisite aggregated sync contribution selections. This indicates the associated prepare sync contribution duty failed. -### Failure Reason: `fetcher_sync_contribution_few_prepares` +### Failure Reason: `sync_contribution_few_prepares` - *Summary*: couldn`t fetch sync contribution due to insufficient partial sync contribution selections - - *Details*: Reason `fetcher_sync_contribution_few_prepares` indicates a sync contribution duty failed in the fetcher step since it couldn`t fetch the prerequisite aggregated sync contribution selections. This indicates the associated prepare sync contribution duty failed due to insufficient partial sync contribution selections submitted by the cluster validator clients. + - *Details*: Reason `sync_contribution_few_prepares` indicates a sync contribution duty failed in the fetcher step since it couldn`t fetch the prerequisite aggregated sync contribution selections. This indicates the associated prepare sync contribution duty failed due to insufficient partial sync contribution selections submitted by the cluster validator clients. -### Failure Reason: `fetcher_sync_contribution_no_external_prepares` +### Failure Reason: `sync_contribution_no_external_prepares` - *Summary*: couldn`t fetch sync contribution due to no partial sync contribution selections received from peers - - *Details*: Reason `fetcher_sync_contribution_no_external_prepares` indicates a sync contribution duty failed in the fetcher step since it couldn`t fetch the prerequisite aggregated sync contribution selections. This indicates the associated prepare sync contribution duty failed due to no partial sync contribution selections received from peers. + - *Details*: Reason `sync_contribution_no_external_prepares` indicates a sync contribution duty failed in the fetcher step since it couldn`t fetch the prerequisite aggregated sync contribution selections. This indicates the associated prepare sync contribution duty failed due to no partial sync contribution selections received from peers. -### Failure Reason: `fetcher_sync_contribution_no_sync_msg` +### Failure Reason: `sync_contribution_no_sync_msg` - *Summary*: couldn`t fetch sync contribution due to failed sync message duty - - *Details*: Reason `fetcher_sync_contribution_no_sync_msg` indicates a sync contribution duty failed in the fetcher step since it couldn`t fetch the prerequisite sync message. This indicates the associated sync message duty failed to obtain a cluster agreed upon value. + - *Details*: Reason `sync_contribution_no_sync_msg` indicates a sync contribution duty failed in the fetcher step since it couldn`t fetch the prerequisite sync message. This indicates the associated sync message duty failed to obtain a cluster agreed upon value. -### Failure Reason: `fetcher_sync_contribution_zero_prepares` +### Failure Reason: `sync_contribution_zero_prepares` - *Summary*: couldn`t fetch sync contribution due to zero partial sync contribution selections - - *Details*: Reason `fetcher_sync_contribution_zero_prepares` indicates a sync contribution duty failed in the fetcher step since it couldn`t fetch the prerequisite aggregated sync contribution selections. This indicates the associated prepare sync contribution duty failed due to no partial sync contribution selections submitted by the cluster validator clients. - -### Failure Reason: `par_sig_db_external` - - *Summary*: bug: failed to store external partial signatures in parsigdb - - *Details*: Reason `par_sig_db_external` indicates a bug in the partial signature database as it is unexpected. - -### Failure Reason: `par_sig_db_inconsistent` - - *Summary*: bug: inconsistent partial signatures received - - *Details*: Reason `par_sig_db_inconsistent` indicates that partial signed data for the duty were inconsistent. This indicates a bug in charon as it is unexpected (for non-sync-committee-duties). - -### Failure Reason: `par_sig_db_inconsistent_sync` - - *Summary*: known limitation: inconsistent sync committee signatures received - - *Details*: Reason `par_sig_db_inconsistent_sync` indicates that partial signed data for the sync committee duty were inconsistent. This is known limitation in this version of charon. - -### Failure Reason: `par_sig_db_insufficient` - - *Summary*: insufficient partial signatures received, minimum required threshold not reached - - *Details*: Reason `par_sig_db_insufficient` indicates that insufficient partial signatures for the duty was received from peers. This indicates problems with peers or p2p network connection problems. - -### Failure Reason: `par_sig_db_internal` - - *Summary*: partial signature database didn`t trigger partial signature exchange, this is unexpected - - *Details*: Reason `par_sig_db_internal` indicates a bug in the partial signature database as it is unexpected. Note this may happen due to expiry race. - -### Failure Reason: `par_sig_ex_receive` - - *Summary*: no partial signatures received from peers - - *Details*: Reason `par_sig_ex_receive` indicates that no partial signature for the duty was received from any peer. This indicates all peers are offline or p2p network connection problems. - -### Failure Reason: `sig_agg` - - *Summary*: bug: threshold aggregation of partial signatures failed due to inconsistent signed data - - *Details*: Reason `sig_agg` indicates that BLS threshold aggregation of sufficient partial signatures failed. This indicates inconsistent signed data. This indicates a bug in charon as it is unexpected. + - *Details*: Reason `sync_contribution_zero_prepares` indicates a sync contribution duty failed in the fetcher step since it couldn`t fetch the prerequisite aggregated sync contribution selections. This indicates the associated prepare sync contribution duty failed due to no partial sync contribution selections submitted by the cluster validator clients. ### Failure Reason: `unknown` - *Summary*: unknown error - *Details*: Reason `unknown` indicates an unknown error occurred. -### Failure Reason: `validator_api` - - *Summary*: signed duty not submitted by local validator client - - *Details*: Reason `validator_api` indicates that partial signature we never submitted by the local validator client. This could indicate that the local validator client is offline, or has connection problems with charon, or has some other problem. See validator client logs for more details. +### Failure Reason: `zero_aggregator_prepares` + - *Summary*: couldn`t aggregate attestation due to zero partial beacon committee selections + - *Details*: Reason `zero_aggregator_prepares` indicates an attestation aggregation duty failed in the fetcher step since it couldn`t fetch the prerequisite aggregated beacon committee selections. This indicates the associated prepare aggregation duty failed due to no partial beacon committee selections submitted by the cluster validator clients.