Skip to content

DTrace updates #1715

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 1 commit into from
May 5, 2025
Merged

DTrace updates #1715

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 8 additions & 3 deletions cmon/src/main.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@ use strum_macros::EnumIter;
use tokio::time::{sleep, Duration};

use crucible::{
Arg, ClientStopReason, ConnectionMode, DsState, NegotiationState,
ClientStopReason, ConnectionMode, DsState, DtraceInfo, NegotiationState,
};

/// Connect to crucible control server
Expand Down Expand Up @@ -272,7 +272,11 @@ fn print_dtrace_header(dd: &[DtraceDisplay]) {

// Print out the values in the dtrace output based on what the DtraceDisplay
// enums are set in the given Vec.
fn print_dtrace_row(d_out: Arg, dd: &[DtraceDisplay], last_job_id: &mut u64) {
fn print_dtrace_row(
d_out: DtraceInfo,
dd: &[DtraceDisplay],
last_job_id: &mut u64,
) {
for display_item in dd.iter() {
match display_item {
DtraceDisplay::State => {
Expand Down Expand Up @@ -412,7 +416,8 @@ fn dtrace_loop(output: Vec<DtraceDisplay>) {
print_dtrace_header(&output);
}
count = (count + 1) % 20;
let d_out: Arg = match serde_json::from_str(&dtrace_out) {
let d_out: DtraceInfo = match serde_json::from_str(&dtrace_out)
{
Ok(a) => a,
Err(e) => {
println!("Err {:?}", e);
Expand Down
5 changes: 3 additions & 2 deletions tools/dtrace/README.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -554,8 +554,9 @@ alan@cat:crucible$ pfexec dtrace -s upstairs_count.d
repair repair repair 1 1 1 0 0 0 0 0 0 0 0 0
```
## upstairs_raw.d
This is a dtrace script that just dumps the `Arg` structure in json format.
The output of this can be sent to other commands for additional processing.
This is a dtrace script that just dumps the `DtraceInfo` structure in json
format. The output of this can be sent to other commands for additional
processing.

## tracegw.d
This is a dtrace example script for counting IOs into and out of
Expand Down
4 changes: 3 additions & 1 deletion tools/dtrace/get-ds-state.d
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@ inline string short_state[string ss] =

crucible_upstairs*:::up-status
{
my_id = json(copyinstr(arg1), "ok.upstairs_id");
my_sesh = json(copyinstr(arg1), "ok.session_id");

this->ds0state = json(copyinstr(arg1), "ok.ds_state[0].type");
Expand All @@ -35,8 +36,9 @@ crucible_upstairs*:::up-status
this->ds2state = json(copyinstr(arg1), "ok.ds_state[2].type");
this->d2 = short_state[this->ds2state];

printf("%6d %8s %3s %3s %3s\n",
printf("%6d %8s %8s %3s %3s %3s\n",
pid,
substr(my_id, 0, 8),
substr(my_sesh, 0, 8),
this->d0,
this->d1,
Expand Down
4 changes: 3 additions & 1 deletion tools/dtrace/get-lr-state.d
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,10 +7,12 @@

crucible_upstairs*:::up-status
{
my_id = json(copyinstr(arg1), "ok.upstairs_id");
my_sesh = json(copyinstr(arg1), "ok.session_id");

printf("%6d %8s %s %s %s %s %s %s\n",
printf("%6d %8s %8s %s %s %s %s %s %s\n",
pid,
substr(my_id, 0, 8),
substr(my_sesh, 0, 8),
json(copyinstr(arg1), "ok.ds_live_repair_completed[0]"),
json(copyinstr(arg1), "ok.ds_live_repair_completed[1]"),
Expand Down
10 changes: 7 additions & 3 deletions tools/dtrace/simple.d
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,8 +16,8 @@ dtrace:::BEGIN
*/
dtrace:::BEGIN, tick-20s
{
printf("%5s %8s ", "PID", "SESSION");
printf("%3s %3s %3s", "DS0", "DS1", "DS2");
printf("%5s %8s %8s", "PID", "UUID", "SESSION");
printf(" %3s %3s %3s", "DS0", "DS1", "DS2");
printf(" %10s %6s %4s", "NEXT_JOB", "DELTA", "CONN");
printf(" %5s %5s", "ELR", "ELC");
printf(" %5s %5s", "ERR", "ERN");
Expand Down Expand Up @@ -57,6 +57,9 @@ crucible_upstairs*:::up-status
this->ds2state = json(copyinstr(arg1), "ok.ds_state[2].type");
this->d2 = short_state[this->ds2state];

this->full_upstairs_id = json(copyinstr(arg1), "ok.upstairs_id");
this->upstairs_id = substr(this->full_upstairs_id, 0, 8);

this->full_session_id = json(copyinstr(arg1), "ok.session_id");
this->session_id = substr(this->full_session_id, 0, 8);

Expand Down Expand Up @@ -88,8 +91,9 @@ crucible_upstairs*:::up-status
strtoll(json(copyinstr(arg1), "ok.ds_extents_confirmed[1]")) +
strtoll(json(copyinstr(arg1), "ok.ds_extents_confirmed[2]"));

printf("%5d %8s %3s %3s %3s %10d %6d %4d %5d %5d %5s %5s\n",
printf("%5d %8s %8s %3s %3s %3s %10d %6d %4d %5d %5d %5s %5s\n",
pid,
this->upstairs_id,
this->session_id,
this->d0,
this->d1,
Expand Down
8 changes: 6 additions & 2 deletions tools/dtrace/single_up_info.d
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@ dtrace:::BEGIN
dtrace:::BEGIN, tick-1s
/show > 20/
{
printf("%8s ", "SESSION");
printf("%8s %8s ", "UUID", "SESSION");
printf("%3s %3s %3s", "DS0", "DS1", "DS2");
printf(" %5s %5s %10s %6s", "UPW", "DSW", "NEXT_JOB", "DELTA");
printf(" %10s", "WRITE_BO");
Expand Down Expand Up @@ -70,6 +70,9 @@ crucible_upstairs*:::up-status
this->ds2state = json(copyinstr(arg1), "ok.ds_state[2].type");
this->d2 = short_state[this->ds2state];

this->full_upstairs_id = json(copyinstr(arg1), "ok.upstairs_id");
this->upstairs_id = substr(this->full_upstairs_id, 0, 8);

this->full_session_id = json(copyinstr(arg1), "ok.session_id");
this->session_id = substr(this->full_session_id, 0, 8);

Expand All @@ -84,8 +87,9 @@ crucible_upstairs*:::up-status
* I'm not very happy about this, but if we don't print it all on one
* line, then multiple sessions will clobber each others output.
*/
printf("%8s %3s %3s %3s %5s %5s %10s %6d %10s %5s %5s %5s %5s %5s %5s %5s %5s %5s %5s %5s %5s %5s %5s %5s\n",
printf("%8s %8s %3s %3s %3s %5s %5s %10s %6d %10s %5s %5s %5s %5s %5s %5s %5s %5s %5s %5s %5s %5s %5s %5s %5s\n",

this->upstairs_id,
this->session_id,
/*
* State for the three downstairs
Expand Down
41 changes: 35 additions & 6 deletions tools/dtrace/sled_upstairs_info.d
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,8 +23,8 @@ dtrace:::BEGIN
tick-1s
/show > 20/
{
printf("%5s %8s ", "PID", "SESSION");
printf("%17s %17s %17s", "DS STATE 0", "DS STATE 1", "DS STATE 2");
printf("%5s %8s %8s ", "PID", "UUID", "SESSION");
printf("%3s %3s %3s", "DS0", "DS1", "DS2");
printf(" %5s %5s %9s", "UPW", "DSW", "NEXT_JOB");
printf(" %10s", "WRITE_BO");
printf(" %5s %5s %5s", "IP0", "IP1", "IP2");
Expand All @@ -36,24 +36,53 @@ tick-1s
show = 0;
}

/*
* Translate the longer state string into a shorter version
*/
inline string short_state[string ss] =
ss == "active" ? "ACT" :
ss == "new" ? "NEW" :
ss == "live_repair_ready" ? "LRR" :
ss == "live_repair" ? "LR" :
ss == "faulted" ? "FLT" :
ss == "offline" ? "OFL" :
ss == "reconcile" ? "REC" :
ss == "wait_quorum" ? "WQ" :
ss == "wait_active" ? "WA" :
ss == "replaced" ? "RPL" :
ss == "connecting" ? "CON" :
ss;

crucible_upstairs*:::up-status
{
show = show + 1;
this->upstairs_id = json(copyinstr(arg1), "ok.upstairs_id");
this->session_id = json(copyinstr(arg1), "ok.session_id");

this->ds0state = json(copyinstr(arg1), "ok.ds_state[0].type");
this->d0 = short_state[this->ds0state];

this->ds1state = json(copyinstr(arg1), "ok.ds_state[1].type");
this->d1 = short_state[this->ds1state];

this->ds2state = json(copyinstr(arg1), "ok.ds_state[2].type");
this->d2 = short_state[this->ds2state];


/*
* I'm not very happy about this very long muli-line printf, but if
* we don't print it all on one line, then multiple sessions will
* clobber each others output.
*/
printf("%5d %8s %17s %17s %17s %5s %5s %9s %10s %5s %5s %5s %5s %5s %5s %5s %5s %5s %5s %5s %5s %5s %5s %5s\n",
printf("%5d %8s %8s %3s %3s %3s %5s %5s %9s %10s %5s %5s %5s %5s %5s %5s %5s %5s %5s %5s %5s %5s %5s %5s %5s\n",
pid,
substr(this->upstairs_id, 0, 8),
substr(this->session_id, 0, 8),

/* State for the three downstairs */
json(copyinstr(arg1), "ok.ds_state[0]"),
json(copyinstr(arg1), "ok.ds_state[1]"),
json(copyinstr(arg1), "ok.ds_state[2]"),
this->d0,
this->d1,
this->d1,

/* Work queue counts for Upstairs and Downstairs */
json(copyinstr(arg1), "ok.up_count"),
Expand Down
8 changes: 6 additions & 2 deletions tools/dtrace/up-info.d
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@ dtrace:::BEGIN,
tick-1s
/show > 20/
{
printf("%5s %8s ", "PID", "SESSION");
printf("%5s %8s %8s ", "PID", "UUID", "SESSION");
printf("%3s %3s %3s", "DS0", "DS1", "DS2");
printf(" %10s %6s %4s", "NEXT_JOB", "DELTA", "CONN");
printf(" %5s %5s", "ELR", "ELC");
Expand Down Expand Up @@ -74,6 +74,9 @@ crucible_upstairs*:::up-status
this->ds2state = json(copyinstr(arg1), "ok.ds_state[2].type");
this->d2 = short_state[this->ds2state];

this->full_upstairs_id = json(copyinstr(arg1), "ok.upstairs_id");
this->upstairs_id = substr(this->full_upstairs_id, 0, 8);

this->full_session_id = json(copyinstr(arg1), "ok.session_id");
this->session_id = substr(this->full_session_id, 0, 8);

Expand Down Expand Up @@ -105,8 +108,9 @@ crucible_upstairs*:::up-status
strtoll(json(copyinstr(arg1), "ok.ds_extents_confirmed[1]")) +
strtoll(json(copyinstr(arg1), "ok.ds_extents_confirmed[2]"));

printf("%5d %8s %3s %3s %3s %10d %6d %4d %5d %5d %5s %5s\n",
printf("%5d %8s %8s %3s %3s %3s %10d %6d %4d %5d %5d %5s %5s\n",
pid,
this->upstairs_id,
this->session_id,
this->d0,
this->d1,
Expand Down
8 changes: 5 additions & 3 deletions upstairs/src/lib.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -321,8 +321,8 @@ pub type CrucibleBlockIOFuture<'a> = Pin<
/// volume__*__done: An IO is completed at the volume layer.
#[usdt::provider(provider = "crucible_upstairs")]
mod cdt {
use crate::Arg;
fn up__status(_: String, arg: Arg) {}
use crate::DtraceInfo;
fn up__status(_: String, arg: DtraceInfo) {}
fn ds__ping__sent(_: u64, _: u8) {}
fn up__apply(_: u64) {}
fn up__action_downstairs(_: u64) {}
Expand Down Expand Up @@ -1508,7 +1508,9 @@ pub(crate) enum BlockOp {
* Stat counters struct used by DTrace
*/
#[derive(Clone, Debug, Serialize, Deserialize)]
pub struct Arg {
pub struct DtraceInfo {
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

RIP to Arg, but this is much more clear

/// Upstairs UUID
pub upstairs_id: String,
/// Upstairs session UUID
pub session_id: String,
/// Jobs on the upstairs guest work queue.
Expand Down
3 changes: 2 additions & 1 deletion upstairs/src/upstairs.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -770,7 +770,8 @@ impl Upstairs {
/// Fires the `up-status` DTrace probe
fn on_stat_update(&self) {
cdt::up__status!(|| {
let arg = Arg {
let arg = DtraceInfo {
upstairs_id: self.cfg.upstairs_id.to_string(),
session_id: self.cfg.session_id.to_string(),
up_count: self.downstairs.gw_active.len() as u32,
up_counters: self.counters,
Expand Down