diff --git a/tools/dtrace/get-up-state.d b/tools/dtrace/get-up-state.d new file mode 100755 index 000000000..f86009e8b --- /dev/null +++ b/tools/dtrace/get-up-state.d @@ -0,0 +1,141 @@ +/* + * Display Upstairs status for all matching processes + */ +#pragma D option quiet +#pragma D option strsize=1k + +/* + * Print the header right away + */ +dtrace:::BEGIN +{ + /* + * We have to init something for last_id so we can use the + * default values for all the session IDs that we don't yet have. + */ + last_id["string"] = (int64_t)1; + printf("%5s %8s ", "PID", "SESSION"); + printf("%3s %3s %3s", "DS0", "DS1", "DS2"); + printf(" %10s %5s %4s", "NEXT_JOB", "DELTA", "CONN"); + printf(" %5s %5s", "ELR", "ELC"); + printf(" %5s %5s", "ERR", "ERN"); + printf("\n"); +} + +/* + * After reporting for 10 seconds, exit + */ +tick-10s +{ + exit(0); +} + +/* + * All variables should be this-> + * Otherwise, there is a chance another probe will fire and + * clobber the contents. + */ +crucible_upstairs*:::up-status +{ + this->ds0state = json(copyinstr(arg1), "ok.ds_state[0]"); + if (this->ds0state == "active") { + this->d0 = "ACT"; + } else if (this->ds0state == "new") { + this->d0 = "NEW"; + } else if (this->ds0state == "live_repair_ready") { + this->d0 = "LRR"; + } else if (this->ds0state == "live_repair") { + this->d0 = " LR"; + } else if (this->ds0state == "faulted") { + this->d0 = "FLT"; + } else if (this->ds0state == "offline") { + this->d0 = "OFL"; + } else { + this->d0 = this->ds0state; + } + + this->ds1state = json(copyinstr(arg1), "ok.ds_state[1]"); + if (this->ds1state == "active") { + this->d1 = "ACT"; + } else if (this->ds1state == "new") { + this->d1 = "NEW"; + } else if (this->ds1state == "live_repair_ready") { + this->d1 = "LRR"; + } else if (this->ds1state == "live_repair") { + this->d1 = " LR"; + } else if (this->ds1state == "faulted") { + this->d1 = "FLT"; + } else if (this->ds1state == "offline") { + this->d1 = "OFL"; + } else { + this->d1 = this->ds1state; + } + + this->ds2state = json(copyinstr(arg1), "ok.ds_state[2]"); + if (this->ds2state == "active") { + this->d2 = "ACT"; + } else if (this->ds2state == "new") { + this->d2 = "NEW"; + } else if (this->ds2state == "live_repair_ready") { + this->d2 = "LRR"; + } else if (this->ds2state == "live_repair") { + this->d2 = " LR"; + } else if (this->ds2state == "faulted") { + this->d2 = "FLT"; + } else if (this->ds2state == "offline") { + this->d2 = "OFL"; + } else { + this->d2 = this->ds2state; + } + + /* + * All these local variables require the "this->" so the probe firing + * from different sessions don't collide with each other. + */ + this->full_session_id = json(copyinstr(arg1), "ok.session_id"); + this->session_id = substr(this->full_session_id, 0, 8); + + this->next_id_str = json(copyinstr(arg1), "ok.next_job_id"); + this->next_id_value = strtoll(this->next_id_str); + + if (last_id[this->session_id] == 0) { + this->delta = 0; + last_id[this->session_id] = this->next_id_value; + } else { + this->delta = this->next_id_value - last_id[this->session_id]; + } + + /* Total of extents live repaired */ + this->elr = strtoll(json(copyinstr(arg1), "ok.ds_extents_repaired[0]")) + + strtoll(json(copyinstr(arg1), "ok.ds_extents_repaired[1]")) + + strtoll(json(copyinstr(arg1), "ok.ds_extents_repaired[2]")); + /* Total of extents not needing repair during live repair */ + this->elc = strtoll(json(copyinstr(arg1), "ok.ds_extents_confirmed[0]")) + + strtoll(json(copyinstr(arg1), "ok.ds_extents_confirmed[1]")) + + strtoll(json(copyinstr(arg1), "ok.ds_extents_confirmed[2]")); + + this->connections = strtoll(json(copyinstr(arg1), "ok.ds_connected[0]")) + + strtoll(json(copyinstr(arg1), "ok.ds_connected[1]")) + + strtoll(json(copyinstr(arg1), "ok.ds_connected[2]")); + + printf("%5d %8s %3s %3s %3s %10s %5d %4d %5d %5d %5s %5s\n", + pid, + this->session_id, + /* + * State for the three downstairs + */ + this->d0, + this->d1, + this->d2, + + /* + * Job ID, job delta and write bytes outstanding + */ + json(copyinstr(arg1), "ok.next_job_id"), + this->delta, + this->connections, + this->elr, + this->elc, + json(copyinstr(arg1), "ok.ds_reconciled"), + json(copyinstr(arg1), "ok.ds_reconcile_needed")); +} diff --git a/tools/dtrace/get-up-state.sh b/tools/dtrace/get-up-state.sh new file mode 100755 index 000000000..2f1efaf27 --- /dev/null +++ b/tools/dtrace/get-up-state.sh @@ -0,0 +1,19 @@ +#!/bin/bash + +filename='/tmp/get-up-state.out' +final='/tmp/get-up-state.final' +rm -f $final + +# Gather our output first. +dtrace -s /opt/oxide/crucible_dtrace/get-up-state.d | awk 'NF' > "$filename" + +# For each session we find, get the latest line and store that in +# the result file. +for id in $(cat $filename | grep -v SESSION | awk '{print $2}' | sort -n | uniq); do + # Find our session, then print the final line + grep "$id" "$filename" | tail -1 >> $final +done +# Print the header +grep "SESSION" "$filename" +# Sort our result by PID and print it out. +sort -n < $final diff --git a/tools/dtrace/upstairs_count.d b/tools/dtrace/upstairs_count.d index e36396d0d..6954e3440 100755 --- a/tools/dtrace/upstairs_count.d +++ b/tools/dtrace/upstairs_count.d @@ -59,6 +59,18 @@ crucible_upstairs*:::gw-write-unwritten-done @write_unwritten_done = count(); } +crucible_upstairs*:::gw-barrier-start +/pid == $1/ +{ + @barrier_start = count(); +} + +crucible_upstairs*:::gw-barrier-done +/pid == $1/ +{ + @barrier_done = count(); +} + /* * Every second, check and see if we have printed enough that it is * time to print the header again @@ -66,17 +78,18 @@ crucible_upstairs*:::gw-write-unwritten-done tick-1s /show > 20/ { - printf("%4s %4s %4s %4s %5s %5s %4s %4s", - "F>", "F<", "W>", "W<", "R>", "R<", "WU>", "WU<"); + printf("%4s %4s %4s %4s %5s %5s %4s %4s %4s %4s", + "F>", "F<", "W>", "W<", "R>", "R<", "WU>", "WU<", "B>", "B<"); printf("\n"); show = 0; } tick-1s { - printa("%@4u %@4u %@4u %@4u %@5u %@5u %@4u %@4u", + printa("%@4u %@4u %@4u %@4u %@5u %@5u %@4u %@4u %@4u %@4u", @flush_start, @flush_done, @write_start, @write_done, - @read_start, @read_done, @write_unwritten_start, @write_unwritten_done + @read_start, @read_done, @write_unwritten_start, @write_unwritten_done, + @barrier_start, @barrier_done ); printf("\n"); clear(@flush_start); @@ -87,5 +100,7 @@ tick-1s clear(@read_done); clear(@write_unwritten_start); clear(@write_unwritten_done); + clear(@barrier_start); + clear(@barrier_done); show = show + 1; } diff --git a/tools/make-dtrace.sh b/tools/make-dtrace.sh index d1c4aba64..d1797471d 100755 --- a/tools/make-dtrace.sh +++ b/tools/make-dtrace.sh @@ -27,6 +27,8 @@ tar cvf ../../out/crucible-dtrace.tar \ get-ds-state.sh \ get-lr-state.d \ get-lr-state.sh \ + get-up-state.d \ + get-up-state.sh \ perf-downstairs-os.d \ perf-downstairs-three.d \ perf-downstairs-tick.d \