Merge branch 'main' into alan/change-region-to-disk

cmon/src/main.rs

@@ -41,7 +41,6 @@ enum DtraceDisplay {
     Replaced,
     ExtentLiveRepair,
     ExtentLimit,
-    Backpressure,
     NextJobId,
     JobDelta,
     DsDelay,
@@ -61,7 +60,6 @@ impl fmt::Display for DtraceDisplay {
             DtraceDisplay::Replaced => write!(f, "replaced"),
             DtraceDisplay::ExtentLiveRepair => write!(f, "extent_live_repair"),
             DtraceDisplay::ExtentLimit => write!(f, "extent_under_repair"),
-            DtraceDisplay::Backpressure => write!(f, "backpressure"),
             DtraceDisplay::NextJobId => write!(f, "next_job_id"),
             DtraceDisplay::JobDelta => write!(f, "job_delta"),
             DtraceDisplay::DsDelay => write!(f, "ds_delay"),
@@ -229,9 +227,6 @@ fn print_dtrace_header(dd: &[DtraceDisplay]) {
             DtraceDisplay::ExtentLimit => {
                 print!(" {:>4}", "EXTL");
             }
-            DtraceDisplay::Backpressure => {
-                print!(" {:>5}", "BAKPR");
-            }
             DtraceDisplay::NextJobId => {
                 print!(" {:>7}", "NEXTJOB");
             }
@@ -348,9 +343,6 @@ fn print_dtrace_row(d_out: Arg, dd: &[DtraceDisplay], last_job_id: &mut u64) {
             DtraceDisplay::ExtentLimit => {
                 print!(" {:4}", d_out.ds_extent_limit);
             }
-            DtraceDisplay::Backpressure => {
-                print!(" {:>5}", d_out.up_backpressure);
-            }
             DtraceDisplay::NextJobId => {
                 print!(" {:>7}", d_out.next_job_id);
             }

tools/dtrace/get-ds-state.d

@@ -0,0 +1,23 @@
+/*
+ * Print a status line for all matching probes.
+ * Exit after 5 seconds.
+ */
+#pragma D option quiet
+#pragma D option strsize=1k
+
+crucible_upstairs*:::up-status
+{
+    my_sesh = json(copyinstr(arg1), "ok.session_id");
+
+    printf("%6d %8s %17s %17s %17s\n",
+        pid,
+        substr(my_sesh, 0, 8),
+        json(copyinstr(arg1), "ok.ds_state[0]"),
+        json(copyinstr(arg1), "ok.ds_state[1]"),
+        json(copyinstr(arg1), "ok.ds_state[2]"));
+}
+
+tick-5s
+{
+    exit(0);
+}

tools/dtrace/get-ds-state.sh

@@ -1,9 +1,16 @@
 #!/bin/bash
 #
-# This script will display the downstairs states for any propolis zones it
-# finds running on a system.
-for zzz in $(zoneadm list | grep propolis); do
-    echo -n "$zzz "
-    ppid=$(zlogin "$zzz" pgrep propolis-server)
-    dtrace -xstrsize=1k -p $ppid -q -n 'crucible_upstairs*:::up-status { printf("%6d %17s %17s %17s", pid, json(copyinstr(arg1), "ok.ds_state[0]"), json(copyinstr(arg1), "ok.ds_state[1]"), json(copyinstr(arg1), "ok.ds_state[2]")); exit(0); }'
-done
+# This script will display the downstairs states for each pid/session
+# it finds running on a system.
+filename='/tmp/get-ds-state.out'
+
+# Gather state on all running propolis servers, record summary to a file
+dtrace -s /opt/oxide/crucible_dtrace/get-ds-state.d | sort -n | uniq | awk 'NF' > "$filename"
+# Walk the lines in the file, append the zone name to each line.
+while read -r p; do
+        # For each line in the file, pull out the PID we are looking at and
+        # print the zone that process is running in.
+        pid=$(echo $p | awk '{print $1}')
+        zone=$(ps -o zone -p $pid | tail -1 | cut -c 1-28)
+        echo "$zone $p"
+done < "$filename"

tools/dtrace/get-lr-state.d

@@ -0,0 +1,26 @@
+/*
+ * Print a live repair status line for all matching probes.
+ * Exit after 5 seconds.
+ */
+#pragma D option quiet
+#pragma D option strsize=1k
+
+crucible_upstairs*:::up-status
+{
+    my_sesh = json(copyinstr(arg1), "ok.session_id");
+
+    printf("%6d %8s %s %s %s %s %s %s\n",
+        pid,
+        substr(my_sesh, 0, 8),
+        json(copyinstr(arg1), "ok.ds_live_repair_completed[0]"),
+        json(copyinstr(arg1), "ok.ds_live_repair_completed[1]"),
+        json(copyinstr(arg1), "ok.ds_live_repair_completed[2]"),
+        json(copyinstr(arg1), "ok.ds_live_repair_aborted[0]"),
+        json(copyinstr(arg1), "ok.ds_live_repair_aborted[1]"),
+        json(copyinstr(arg1), "ok.ds_live_repair_aborted[2]"));
+}
+
+tick-5s
+{
+    exit(0);
+}

tools/dtrace/get-lr-state.sh

@@ -1,9 +1,16 @@
 #!/bin/bash
-#
-# This script will log into every propolis zone it finds and get the
-# DTrace live repair counters from propolis-server in each zone.
-for zzz in $(zoneadm list | grep propolis); do
-    echo -n "$zzz "
-    ppid=$(zlogin "$zzz" pgrep propolis-server)
-    dtrace -xstrsize=1k -p $ppid -q -n 'crucible_upstairs*:::up-status { printf("%6d %s %s %s %s %s %s", pid, json(copyinstr(arg1), "ok.ds_live_repair_completed[0]"), json(copyinstr(arg1), "ok.ds_live_repair_completed[1]"), json(copyinstr(arg1), "ok.ds_live_repair_completed[2]"), json(copyinstr(arg1), "ok.ds_live_repair_aborted[0]"), json(copyinstr(arg1), "ok.ds_live_repair_aborted[1]"), json(copyinstr(arg1), "ok.ds_live_repair_aborted[2]")); exit(0); }'
-done
+
+# This script will display the downstairs live repair for each
+# pid/session it finds running on a system.
+filename='/tmp/get-lr-state.out'
+
+# Gather state on all running propolis servers, record summary to a file
+dtrace -s /opt/oxide/crucible_dtrace/get-lr-state.d | sort -n | uniq | awk 'NF' > "$filename"
+# Walk the lines in the file, append the zone name to each line.
+while read -r p; do
+        # For each line in the file, pull out the PID we are looking at and
+        # print the zone that process is running in.
+        pid=$(echo $p | awk '{print $1}')
+        zone=$(ps -o zone -p $pid | tail -1 | cut -c 1-28)
+        echo "$zone $p"
+done < "$filename"

tools/dtrace/single_up_info.d

@@ -41,7 +41,7 @@ crucible_upstairs*:::up-status
      * I'm not very happy about this, but if we don't print it all on one
      * line, then multiple sessions will clobber each others output.
      */
-    printf("%8s %17s %17s %17s %5s %5s %9s %5s %10s  %5s %5s %5s  %5s %5s %5s  %5s %5s %5s  %5s %5s %5s  %5s %5s %5s\n",
+    printf("%8s %17s %17s %17s %5s %5s %5s %10s  %5s %5s %5s  %5s %5s %5s  %5s %5s %5s  %5s %5s %5s  %5s %5s %5s\n",
 
     substr(session_id, 0, 8),
 
@@ -62,7 +62,6 @@ crucible_upstairs*:::up-status
      * Job ID delta and backpressure
      */
     json(copyinstr(arg1), "ok.next_job_id"),
-    json(copyinstr(arg1), "ok.up_backpressure"),
     json(copyinstr(arg1), "ok.write_bytes_out"),
 
     /*

tools/dtrace/sled_upstairs_info.d

@@ -46,8 +46,7 @@ crucible_upstairs*:::up-status
      * we don't print it all on one line, then multiple sessions will
      * clobber each others output.
      */
-    printf("%5d %8s %17s %17s %17s %5s %5s %9s %5s %10s  %5s %5s %5s  %5s %5s %5s  %5s %5s %5s  %5s %5s %5s  %5s %5s %5s  %5s %5s %5s\n",
-
+    printf("%5d %8s %17s %17s %17s %5s %5s %5s %10s  %5s %5s %5s  %5s %5s %5s  %5s %5s %5s  %5s %5s %5s  %5s %5s %5s\n",
     pid,
     substr(session_id, 0, 8),
 
@@ -62,7 +61,6 @@ crucible_upstairs*:::up-status
 
     /* Job ID and backpressure */
     json(copyinstr(arg1), "ok.next_job_id"),
-    json(copyinstr(arg1), "ok.up_backpressure"),
     json(copyinstr(arg1), "ok.write_bytes_out"),
 
     /* In progress jobs on the work list for each downstairs */

tools/dtrace/upstairs_info.d

@@ -21,7 +21,7 @@ tick-1s
 {
     printf("%6s ", "PID");
     printf("%17s %17s %17s", "DS STATE 0", "DS STATE 1", "DS STATE 2");
-    printf("  %5s %5s %9s %5s", "UPW", "DSW", "JOBID", "BAKPR");
+    printf("  %5s %5s %9s", "UPW", "DSW", "JOBID");
     printf(" %10s", "WRITE_BO");
     printf("  %5s %5s %5s", "IP0", "IP1", "IP2");
     printf("  %5s %5s %5s", "D0", "D1", "D2");
@@ -49,10 +49,9 @@ crucible_upstairs*:::up-status
     printf(" %5s", json(copyinstr(arg1), "ok.ds_count"));
 
     /*
-     * Job ID and backpressure
+     * Job ID and outstanding bytes
      */
     printf(" %9s", json(copyinstr(arg1), "ok.next_job_id"));
-    printf(" %5s", json(copyinstr(arg1), "ok.up_backpressure"));
     printf(" %10s", json(copyinstr(arg1), "ok.write_bytes_out"));
 
     /*

tools/make-dtrace.sh

@@ -23,7 +23,9 @@ tar cvf ../../out/crucible-dtrace.tar \
     README.md \
     all_downstairs.d \
     downstairs_count.d \
+    get-ds-state.d \
     get-ds-state.sh \
+    get-lr-state.d \
     get-lr-state.sh \
     perf-downstairs-os.d \
     perf-downstairs-three.d \