Fix dtrace system level scripts

The dtrace scripts did not handle multiple session IDs.
Refactor the scripts get-ds-state and get-lr-state to correctly
handle a PID with multiple sessions and improved the printing time
for getting a system summary output.

tools/dtrace/get-ds-state.d

@@ -0,0 +1,23 @@
+/*
+ * Print a status line for a given PID.
+ * Exit after 5 seconds.
+ */
+#pragma D option quiet
+#pragma D option strsize=1k
+
+crucible_upstairs*:::up-status
+{
+    my_sesh = json(copyinstr(arg1), "ok.session_id");
+
+    printf("%6d %8s %17s %17s %17s\n",
+        pid,
+        substr(my_sesh, 0, 8),
+        json(copyinstr(arg1), "ok.ds_state[0]"),
+        json(copyinstr(arg1), "ok.ds_state[1]"),
+        json(copyinstr(arg1), "ok.ds_state[2]"));
+}
+
+tick-5s
+{
+    exit(0);
+}

tools/dtrace/get-ds-state.sh

@@ -1,9 +1,16 @@
 #!/bin/bash
 #
-# This script will display the downstairs states for any propolis zones it
-# finds running on a system.
-for zzz in $(zoneadm list | grep propolis); do
-    echo -n "$zzz "
-    ppid=$(zlogin "$zzz" pgrep propolis-server)
-    dtrace -xstrsize=1k -p $ppid -q -n 'crucible_upstairs*:::up-status { printf("%6d %17s %17s %17s", pid, json(copyinstr(arg1), "ok.ds_state[0]"), json(copyinstr(arg1), "ok.ds_state[1]"), json(copyinstr(arg1), "ok.ds_state[2]")); exit(0); }'
-done
+# This script will display the downstairs states for each pid/session
+# it finds running on a system.
+filename='/tmp/get-ds-state.out'
+
+# Gather state on all running propolis servers, record summary to a file
+dtrace -s /opt/oxide/crucible_dtrace/get-ds-state.d | sort -n | uniq | awk 'NF' > "$filename"
+# Walk the lines in the file, append the zone name to each line.
+while read p; do
+        # For each line in the file, pull out the PID we are looking at and
+        # print the zone that process is running in.
+        pid=$(echo $p | awk '{print $1}')
+        zone=$(ps -o zone -p $pid | tail -1 | cut -c 1-28)
+        echo "$zone $p"
+done < "$filename"

tools/dtrace/get-lr-state.d

@@ -0,0 +1,26 @@
+/*
+ * Print a live reapir status line.
+ * Exit after 5 seconds.
+ */
+#pragma D option quiet
+#pragma D option strsize=1k
+
+crucible_upstairs*:::up-status
+{
+    my_sesh = json(copyinstr(arg1), "ok.session_id");
+
+    printf("%6d %8s %s %s %s %s %s %s\n",
+        pid,
+        substr(my_sesh, 0, 8),
+        json(copyinstr(arg1), "ok.ds_live_repair_completed[0]"),
+        json(copyinstr(arg1), "ok.ds_live_repair_completed[1]"),
+        json(copyinstr(arg1), "ok.ds_live_repair_completed[2]"),
+        json(copyinstr(arg1), "ok.ds_live_repair_aborted[0]"),
+        json(copyinstr(arg1), "ok.ds_live_repair_aborted[1]"),
+        json(copyinstr(arg1), "ok.ds_live_repair_aborted[2]"));
+}
+
+tick-5s
+{
+    exit(0);
+}

tools/dtrace/get-lr-state.sh

@@ -1,9 +1,16 @@
 #!/bin/bash
-#
-# This script will log into every propolis zone it finds and get the
-# DTrace live repair counters from propolis-server in each zone.
-for zzz in $(zoneadm list | grep propolis); do
-    echo -n "$zzz "
-    ppid=$(zlogin "$zzz" pgrep propolis-server)
-    dtrace -xstrsize=1k -p $ppid -q -n 'crucible_upstairs*:::up-status { printf("%6d %s %s %s %s %s %s", pid, json(copyinstr(arg1), "ok.ds_live_repair_completed[0]"), json(copyinstr(arg1), "ok.ds_live_repair_completed[1]"), json(copyinstr(arg1), "ok.ds_live_repair_completed[2]"), json(copyinstr(arg1), "ok.ds_live_repair_aborted[0]"), json(copyinstr(arg1), "ok.ds_live_repair_aborted[1]"), json(copyinstr(arg1), "ok.ds_live_repair_aborted[2]")); exit(0); }'
-done
+
+# This script will display the downstairs live repair for each
+# pid/session it finds running on a system.
+filename='/tmp/get-lr-state.out'
+
+# Gather state on all running propolis servers, record summary to a file
+dtrace -s /opt/oxide/crucible_dtrace/get-lr-state.d | sort -n | uniq | awk 'NF' > "$filename"
+# Walk the lines in the file, append the zone name to each line.
+while read p; do
+        # For each line in the file, pull out the PID we are looking at and
+        # print the zone that process is running in.
+        pid=$(echo $p | awk '{print $1}')
+        zone=$(ps -o zone -p $pid | tail -1 | cut -c 1-28)
+        echo "$zone $p"
+done < "$filename"

tools/make-dtrace.sh

@@ -23,7 +23,9 @@ tar cvf ../../out/crucible-dtrace.tar \
     README.md \
     all_downstairs.d \
     downstairs_count.d \
+    get-ds-state.d \
     get-ds-state.sh \
+    get-lr-state.d \
     get-lr-state.sh \
     perf-downstairs-os.d \
     perf-downstairs-three.d \