From ef090e131e14d6a67a4c2112d760c74c83129606 Mon Sep 17 00:00:00 2001 From: Evan Chan Date: Sat, 29 Oct 2022 21:28:19 -0700 Subject: [PATCH] Add custom Ying memory profiler and prevent giant allocations (#5649) * Experiment: add TaiAllocator * Remove Ying profiler from stress and put it in sui-node; remove old jemalloc profiling code * Clippy, workspace-hack, misc * Update pointer to ying profiler --- Cargo.lock | 42 +++++++++++++- Cargo.toml | 11 ++-- crates/sui-benchmark/src/bin/stress.rs | 1 + crates/sui-node/Cargo.toml | 5 +- crates/sui-node/src/main.rs | 80 ++++---------------------- crates/workspace-hack/Cargo.toml | 6 ++ docker/sui-node/Dockerfile | 4 -- 7 files changed, 65 insertions(+), 84 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 02f3c23474036..76e8f47ec7db7 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1184,6 +1184,18 @@ dependencies = [ "winapi", ] +[[package]] +name = "coarsetime" +version = "0.1.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "454038500439e141804c655b4cd1bc6a70bcb95cd2bc9463af5661b6956f0e46" +dependencies = [ + "libc", + "once_cell", + "wasi 0.11.0+wasi-snapshot-preview1", + "wasm-bindgen", +] + [[package]] name = "codespan" version = "0.11.1" @@ -8345,7 +8357,6 @@ dependencies = [ "chrono", "clap 3.2.22", "futures", - "jemalloc-ctl", "multiaddr", "mysten-network", "narwhal-network", @@ -8365,6 +8376,7 @@ dependencies = [ "tracing", "typed-store", "workspace-hack", + "ying-profiler", ] [[package]] @@ -10337,6 +10349,7 @@ dependencies = [ "clap_lex", "clear_on_drop", "clipboard-win", + "coarsetime", "codespan", "codespan-reporting", "collectable", @@ -10996,16 +11009,27 @@ dependencies = [ "windows-sys", "windows_x86_64_msvc", "winreg", + "wyhash", "wyz", "x509-parser", "yaml-rust", "yansi", "yasna", + "ying-profiler", "zeroize", "zeroize_derive", "zstd-sys", ] +[[package]] +name = "wyhash" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "baf6e163c25e3fac820b4b453185ea2dea3b6a3e0a721d4d23d75bd33734c295" +dependencies = [ + "rand_core 0.6.4", +] + [[package]] name = "wyz" version = "0.2.0" @@ -11064,6 +11088,22 @@ dependencies = [ "time 0.3.15", ] +[[package]] +name = "ying-profiler" +version = "0.1.0" +source = "git+https://github.com/velvia/ying-profiler#1a3e476299bfc1ff4a3168edd9fcf1ccda521aea" +dependencies = [ + "backtrace", + "chrono", + "coarsetime", + "dashmap", + "once_cell", + "rand 0.8.5", + "regex", + "tracing", + "wyhash", +] + [[package]] name = "zeroize" version = "1.5.7" diff --git a/Cargo.toml b/Cargo.toml index 25a682727d479..aee8765934af7 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -47,16 +47,15 @@ members = [ version = "0.14.0" [profile.release] -# The following two lines add minimal symbol information, which helps certain profilers like Bytehound -# without significantly increasing binary size -debug = true -strip = 'debuginfo' +# debug = 1 means line charts only, which is minimum needed for good stack traces +debug = 1 +strip = 'none' # Exit process with SIGABRT when any thread panics panic = 'abort' [profile.bench] -# Do not strip any debug info. This helps the widest set of profiling tools -debug = true +# debug = 1 means line charts only, which is minimum needed for good stack traces +debug = 1 strip = 'none' [profile.simulator] diff --git a/crates/sui-benchmark/src/bin/stress.rs b/crates/sui-benchmark/src/bin/stress.rs index c87dc0975cdf1..d95607b90d374 100644 --- a/crates/sui-benchmark/src/bin/stress.rs +++ b/crates/sui-benchmark/src/bin/stress.rs @@ -252,6 +252,7 @@ async fn main() -> Result<()> { config.log_file = Some(opts.log_path); } let _guard = config.with_env().init(); + let registry: Arc = Arc::new(metrics::start_prometheus_server( format!("{}:{}", opts.client_metric_host, opts.client_metric_port) .parse() diff --git a/crates/sui-node/Cargo.toml b/crates/sui-node/Cargo.toml index 16f098c9efcb0..f43c85598ff96 100644 --- a/crates/sui-node/Cargo.toml +++ b/crates/sui-node/Cargo.toml @@ -31,12 +31,11 @@ sui-telemetry = { path = "../sui-telemetry" } sui-types = { path = "../sui-types" } narwhal-network = { path = "../../narwhal/network" } +ying-profiler = { git = "https://github.com/velvia/ying-profiler", features = ["profile-spans"] } + telemetry-subscribers.workspace = true mysten-network.workspace = true workspace-hack.workspace = true [target.'cfg(msim)'.dependencies] sui-simulator = { path = "../sui-simulator" } - -[target.'cfg(not(target_env = "msvc"))'.dependencies] -jemalloc-ctl = "^0.5" diff --git a/crates/sui-node/src/main.rs b/crates/sui-node/src/main.rs index fc2d3447b810b..c7a493018dbe4 100644 --- a/crates/sui-node/src/main.rs +++ b/crates/sui-node/src/main.rs @@ -11,7 +11,7 @@ use sui_node::metrics; use sui_telemetry::send_telemetry_event; use tokio::task; use tokio::time::sleep; -use tracing::{debug, info}; +use tracing::info; #[derive(Parser)] #[clap(rename_all = "kebab-case", version)] @@ -23,19 +23,12 @@ struct Args { listen_address: Option, } -// Memory profiling is now done automatically based on increases in total memory usage. -// Set MALLOC_CONF (when loading jemalloc dynamically) or _RJEM_MALLOC_CONF (when jemalloc is -// linked statically) to: prof:true -// See [doc/src/contribute/observability.md] for more info. -// For more memory profiling info see https://github.com/jemalloc/jemalloc/wiki/Use-Case%3A-Heap-Profiling -#[cfg(not(target_env = "msvc"))] -use jemalloc_ctl::{epoch, stats}; +// Memory profiling is now done automatically by the Ying profiler. +use ying_profiler::utils::ProfilerRunner; +use ying_profiler::YingProfiler; -// Ratio of memory used compared to before that triggers a new profiling dump -const MEMORY_INCREASE_PROFILING_RATIO: f64 = 1.2; -// Interval between checks for memory profile dumps -const MEMORY_PROFILING_INTERVAL_SECS: u64 = 300; -const PROF_DUMP: &[u8] = b"prof.dump\0"; +#[global_allocator] +static YING_ALLOC: YingProfiler = YingProfiler; #[tokio::main] async fn main() -> Result<()> { @@ -59,63 +52,10 @@ async fn main() -> Result<()> { config.network_address = listen_address; } - #[cfg(not(target_env = "msvc"))] - { - use jemalloc_ctl::config; - use std::ffi::CString; - use std::time::Duration; - use tracing::info; - - let malloc_conf = config::malloc_conf::mib().unwrap(); - info!("Default Jemalloc conf: {}", malloc_conf.read().unwrap()); - - std::thread::spawn(|| { - // This is the initial size of memory beyond which profiles are dumped - let mut last_allocated_mb = 100; - loop { - // many statistics are cached and only updated when the epoch is advanced. - epoch::advance().unwrap(); - - // NOTE: The below code does not return values when a malloc-based profiler like Bytehound - // is used. Bytehound does not implement the stat APIs needed. - let allocated = stats::allocated::read().unwrap() / (1024 * 1024); - let resident = stats::resident::read().unwrap() / (1024 * 1024); - info!( - "Jemalloc: {} MB allocated / {} MB resident", - allocated, resident - ); - - // TODO: split this out into mysten-infra so everyone can pick it up - // The reason why we use manual code to dump out profiles is because the automatic JEPROF - // options dump out too often. We really just want profiles when the retained memory - // keeps growing, as we want to know why. - // Setting the timestamp and memory size in the filename helps us pick the profiles - // to use when doing analysis. Default JEPROF profiles just have a counter which is not - // helpful. This helps correlate to time and total memory consumed. - // - // NOTE: One needs to set MALLOC_CONF to `prof:true` for the below to work - if (allocated as f64 / last_allocated_mb as f64) > MEMORY_INCREASE_PROFILING_RATIO { - info!("Significant memory increase registered, dumping profile: new = {}, old = {}", - allocated, last_allocated_mb); - last_allocated_mb = allocated; - - // Formulate profiling filename based on ISO8601 timestamp and number of MBs - let dt = chrono::offset::Local::now(); - let dt_str = dt.to_rfc3339_opts(chrono::SecondsFormat::Secs, true); - let dump_name = format!("jeprof.{}.{}MB.heap", dt_str, allocated); - - // Trigger profiling dump - let dump_name_cstr = CString::new(dump_name).expect("Cannot create dump name"); - unsafe { - if jemalloc_ctl::raw::write(PROF_DUMP, dump_name_cstr.as_ptr()).is_err() { - debug!("Cannot dump memory profile, is _RJEM_MALLOC_CONF set to prof:true?"); - } - } - } - std::thread::sleep(Duration::from_secs(MEMORY_PROFILING_INTERVAL_SECS)); - } - }); - } + // Spins up a thread to check memory usage every minute, and dump out stack traces/profiles + // if it has moved up or down more than 15%. Also allow configuration of dump directory. + let profile_dump_dir = std::env::var("SUI_MEM_PROFILE_DIR").unwrap_or_default(); + ProfilerRunner::new(60, 15, &profile_dump_dir).spawn(); let is_validator = config.consensus_config().is_some(); task::spawn(async move { diff --git a/crates/workspace-hack/Cargo.toml b/crates/workspace-hack/Cargo.toml index 3049488e1053f..c498052436923 100644 --- a/crates/workspace-hack/Cargo.toml +++ b/crates/workspace-hack/Cargo.toml @@ -101,6 +101,7 @@ clap-f595c2ba2a3f28df = { package = "clap", version = "2", features = ["ansi_ter clap-7b89eefb6aaa9bf3 = { package = "clap", version = "3", features = ["atty", "clap_derive", "color", "derive", "once_cell", "std", "strsim", "suggestions", "termcolor"] } clap_lex = { version = "0.2", default-features = false } clear_on_drop = { version = "0.2", default-features = false } +coarsetime = { version = "0.1", default-features = false } codespan = { version = "0.11", default-features = false, features = ["serde", "serialization"] } codespan-reporting = { version = "0.11", default-features = false, features = ["serde", "serialization"] } collectable = { version = "0.0.2", default-features = false } @@ -610,11 +611,13 @@ web-sys = { version = "0.3", default-features = false, features = ["BinaryType", webpki = { version = "0.22", default-features = false, features = ["alloc", "std"] } webpki-roots = { version = "0.22", default-features = false } whoami = { version = "1" } +wyhash = { version = "0.5", default-features = false } wyz = { version = "0.2", default-features = false, features = ["alloc"] } x509-parser = { version = "0.14" } yaml-rust = { version = "0.4", default-features = false } yansi = { version = "0.5", default-features = false } yasna = { version = "0.5", features = ["std", "time"] } +ying-profiler = { git = "https://github.com/velvia/ying-profiler", default-features = false, features = ["profile-spans", "tracing"] } zeroize = { version = "1", features = ["alloc", "zeroize_derive"] } zstd-sys = { version = "2", features = ["legacy", "zdict_builder"] } @@ -724,6 +727,7 @@ clap-7b89eefb6aaa9bf3 = { package = "clap", version = "3", features = ["atty", " clap_derive = { version = "3" } clap_lex = { version = "0.2", default-features = false } clear_on_drop = { version = "0.2", default-features = false } +coarsetime = { version = "0.1", default-features = false } codespan = { version = "0.11", default-features = false, features = ["serde", "serialization"] } codespan-reporting = { version = "0.11", default-features = false, features = ["serde", "serialization"] } collectable = { version = "0.0.2", default-features = false } @@ -1324,11 +1328,13 @@ webpki = { version = "0.22", default-features = false, features = ["alloc", "std webpki-roots = { version = "0.22", default-features = false } which = { version = "4", default-features = false } whoami = { version = "1" } +wyhash = { version = "0.5", default-features = false } wyz = { version = "0.2", default-features = false, features = ["alloc"] } x509-parser = { version = "0.14" } yaml-rust = { version = "0.4", default-features = false } yansi = { version = "0.5", default-features = false } yasna = { version = "0.5", features = ["std", "time"] } +ying-profiler = { git = "https://github.com/velvia/ying-profiler", default-features = false, features = ["profile-spans", "tracing"] } zeroize = { version = "1", features = ["alloc", "zeroize_derive"] } zeroize_derive = { version = "1", default-features = false } zstd-sys = { version = "2", features = ["legacy", "zdict_builder"] } diff --git a/docker/sui-node/Dockerfile b/docker/sui-node/Dockerfile index 2aa9a1298ed89..ec2d9589315d0 100644 --- a/docker/sui-node/Dockerfile +++ b/docker/sui-node/Dockerfile @@ -51,9 +51,5 @@ COPY --from=builder /sui/target/release/sui-node /usr/local/bin ARG BUILD_DATE ARG GIT_REVISION -# Use jemalloc as memory allocator -ENV LD_PRELOAD /usr/lib/x86_64-linux-gnu/libjemalloc.so -# Enable automatic memory profiling. -ENV MALLOC_CONF prof:true,prof_prefix:/data/jeprof.out,lg_prof_interval:36 LABEL build-date=$BUILD_DATE LABEL git-revision=$GIT_REVISION