Skip to content

Commit

Permalink
impl
Browse files Browse the repository at this point in the history
  • Loading branch information
pugachAG committed Feb 27, 2024
1 parent 7bad46a commit 731ff7c
Show file tree
Hide file tree
Showing 75 changed files with 489 additions and 378 deletions.
8 changes: 8 additions & 0 deletions Cargo.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

27 changes: 23 additions & 4 deletions chain/chain/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@ strum.workspace = true
thiserror.workspace = true
tracing.workspace = true
yansi.workspace = true
easy-ext.workspace = true

near-async.workspace = true
near-cache.workspace = true
Expand All @@ -43,8 +44,15 @@ near-performance-metrics-macros.workspace = true
near-pool.workspace = true
near-primitives.workspace = true
near-store.workspace = true
node-runtime.workspace = true
near-parameters.workspace = true
near-vm-runner.workspace = true
near-mainnet-res.workspace = true

[dev-dependencies]
serde_json.workspace = true
primitive-types.workspace = true
tempfile.workspace = true
insta.workspace = true
assert_matches.workspace = true

Expand All @@ -55,7 +63,12 @@ expensive_tests = []
test_features = []
shadow_chunk_validation = []
no_cache = ["near-store/no_cache"]
new_epoch_sync = ["near-store/new_epoch_sync", "near-primitives/new_epoch_sync", "near-epoch-manager/new_epoch_sync", "near-chain-primitives/new_epoch_sync"]
new_epoch_sync = [
"near-store/new_epoch_sync",
"near-primitives/new_epoch_sync",
"near-epoch-manager/new_epoch_sync",
"near-chain-primitives/new_epoch_sync",
]

protocol_feature_reject_blocks_with_outdated_protocol_version = [
"near-primitives/protocol_feature_reject_blocks_with_outdated_protocol_version",
Expand All @@ -66,26 +79,32 @@ nightly = [
"near-chain-configs/nightly",
"near-client-primitives/nightly",
"near-epoch-manager/nightly",
"near-mainnet-res/nightly",
"near-network/nightly",
"near-o11y/nightly",
"near-parameters/nightly",
"near-pool/nightly",
"near-primitives/nightly",
"near-store/nightly",
"near-vm-runner/nightly",
"nightly_protocol",
"node-runtime/nightly",
"protocol_feature_reject_blocks_with_outdated_protocol_version",
]
nightly_protocol = [
"near-async/nightly_protocol",
"near-chain-configs/nightly_protocol",
"near-client-primitives/nightly_protocol",
"near-epoch-manager/nightly_protocol",
"near-mainnet-res/nightly_protocol",
"near-network/nightly_protocol",
"near-o11y/nightly_protocol",
"near-parameters/nightly_protocol",
"near-pool/nightly_protocol",
"near-primitives/nightly_protocol",
"near-store/nightly_protocol",
"near-vm-runner/nightly_protocol",
"node-runtime/nightly_protocol",
]
statelessnet_protocol = [
"near-primitives/statelessnet_protocol",
]
statelessnet_protocol = ["near-primitives/statelessnet_protocol"]
sandbox = ["near-primitives/sandbox"]
1 change: 1 addition & 0 deletions chain/chain/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@ pub mod migrations;
pub mod missing_chunks;
pub mod orphan;
pub mod resharding;
pub mod runtime;
mod state_request_tracker;
pub mod state_snapshot_actor;
mod stateless_validation;
Expand Down
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
use near_chain::near_chain_primitives::error::QueryError;
use crate::near_chain_primitives::error::QueryError;

#[easy_ext::ext(FromStateViewerErrors)]
impl QueryError {
Expand Down
67 changes: 67 additions & 0 deletions chain/chain/src/runtime/metrics.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,67 @@
use near_o11y::metrics::{
exponential_buckets, linear_buckets, processing_time_buckets, try_create_histogram_vec,
try_create_int_gauge_vec, HistogramVec, IntGaugeVec,
};

use once_cell::sync::Lazy;

pub(crate) static APPLY_CHUNK_DELAY: Lazy<HistogramVec> = Lazy::new(|| {
try_create_histogram_vec(
"near_apply_chunk_delay_seconds",
"Time to process a chunk. Gas used by the chunk is a metric label, rounded up to 100 teragas.",
&["tgas_ceiling"],
Some(linear_buckets(0.0, 0.05, 50).unwrap()),
)
.unwrap()
});

pub(crate) static DELAYED_RECEIPTS_COUNT: Lazy<IntGaugeVec> = Lazy::new(|| {
try_create_int_gauge_vec(
"near_delayed_receipts_count",
"The count of the delayed receipts. Indicator of congestion.",
&["shard_id"],
)
.unwrap()
});

pub(crate) static PREPARE_TX_SIZE: Lazy<HistogramVec> = Lazy::new(|| {
try_create_histogram_vec(
"near_prepare_tx_size",
"Sum of transaction sizes per produced chunk, as a histogram",
&["shard_id"],
// Maximum is < 14MB, typical values are unknown right now so buckets
// might need to be adjusted later when we have collected data
Some(vec![1_000.0, 10_000., 100_000., 500_000., 1e6, 2e6, 4e6, 8e6, 12e6]),
)
.unwrap()
});

pub static APPLYING_CHUNKS_TIME: Lazy<HistogramVec> = Lazy::new(|| {
try_create_histogram_vec(
"near_applying_chunks_time",
"Time taken to apply chunks per shard",
&["shard_id"],
Some(processing_time_buckets()),
)
.unwrap()
});

pub(crate) static STATE_SYNC_OBTAIN_PART_DELAY: Lazy<HistogramVec> = Lazy::new(|| {
try_create_histogram_vec(
"near_state_sync_obtain_part_delay_sec",
"Latency of applying a state part",
&["shard_id", "result"],
Some(exponential_buckets(0.001, 2.0, 20).unwrap()),
)
.unwrap()
});

pub(crate) static STATE_SYNC_APPLY_PART_DELAY: Lazy<HistogramVec> = Lazy::new(|| {
try_create_histogram_vec(
"near_state_sync_apply_part_delay_sec",
"Latency of applying a state part",
&["shard_id"],
Some(exponential_buckets(0.001, 2.0, 20).unwrap()),
)
.unwrap()
});
62 changes: 62 additions & 0 deletions chain/chain/src/runtime/migrations.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,62 @@
use near_primitives::receipt::ReceiptResult;
use near_primitives::runtime::migration_data::MigrationData;
use near_primitives::types::Gas;

/// In test runs reads and writes here used 442 TGas, but in test on live net migration take
/// between 4 and 4.5s. We do not want to process any receipts in this block
const GAS_USED_FOR_STORAGE_USAGE_DELTA_MIGRATION: Gas = 1_000_000_000_000_000;

pub fn load_migration_data(chain_id: &str) -> MigrationData {
let is_mainnet = chain_id == near_primitives::chains::MAINNET;
MigrationData {
storage_usage_delta: if is_mainnet {
near_mainnet_res::mainnet_storage_usage_delta()
} else {
Vec::new()
},
storage_usage_fix_gas: if is_mainnet {
GAS_USED_FOR_STORAGE_USAGE_DELTA_MIGRATION
} else {
0
},
restored_receipts: if is_mainnet {
near_mainnet_res::mainnet_restored_receipts()
} else {
ReceiptResult::default()
},
}
}

#[cfg(test)]
mod tests {
use super::*;
use near_mainnet_res::mainnet_restored_receipts;
use near_mainnet_res::mainnet_storage_usage_delta;
use near_primitives::hash::hash;

#[test]
fn test_migration_data() {
assert_eq!(
hash(serde_json::to_string(&mainnet_storage_usage_delta()).unwrap().as_bytes())
.to_string(),
"2fEgaLFBBJZqgLQEvHPsck4NS3sFzsgyKaMDqTw5HVvQ"
);
let mainnet_migration_data = load_migration_data(near_primitives::chains::MAINNET);
assert_eq!(mainnet_migration_data.storage_usage_delta.len(), 3112);
let testnet_migration_data = load_migration_data(near_primitives::chains::TESTNET);
assert_eq!(testnet_migration_data.storage_usage_delta.len(), 0);
}

#[test]
fn test_restored_receipts_data() {
assert_eq!(
hash(serde_json::to_string(&mainnet_restored_receipts()).unwrap().as_bytes())
.to_string(),
"48ZMJukN7RzvyJSW9MJ5XmyQkQFfjy2ZxPRaDMMHqUcT"
);
let mainnet_migration_data = load_migration_data(near_primitives::chains::MAINNET);
assert_eq!(mainnet_migration_data.restored_receipts.get(&0u64).unwrap().len(), 383);
let testnet_migration_data = load_migration_data(near_primitives::chains::TESTNET);
assert!(testnet_migration_data.restored_receipts.is_empty());
}
}
Loading

0 comments on commit 731ff7c

Please sign in to comment.