add super fancy OMDB output

dev-tools/omdb/src/bin/omdb/nexus.rs

@@ -32,6 +32,7 @@ use nexus_client::types::UninitializedSledId;
 use nexus_db_queries::db::lookup::LookupPath;
 use nexus_saga_recovery::LastPass;
 use nexus_types::deployment::Blueprint;
+use nexus_types::internal_api::background::InstanceReincarnationStatus;
 use nexus_types::internal_api::background::LookupRegionPortStatus;
 use nexus_types::internal_api::background::RegionReplacementDriverStatus;
 use nexus_types::internal_api::background::RegionSnapshotReplacementFinishStatus;
@@ -1645,6 +1646,80 @@ fn print_task_details(bgtask: &BackgroundTask, details: &serde_json::Value) {
                 }
             }
         }
+    } else if name == "instance_reincarnation" {
+        match serde_json::from_value::<InstanceReincarnationStatus>(
+            details.clone(),
+        ) {
+            Err(error) => eprintln!(
+                "warning: failed to interpret task details: {:?}: {:?}",
+                error, details
+            ),
+            Ok(InstanceReincarnationStatus {
+                instances_found,
+                instances_reincarnated,
+                already_reincarnated,
+                query_error,
+                restart_errors,
+            }) => {
+                const FOUND: &'static str =
+                    "instances eligible for reincarnation:";
+                const REINCARNATED: &'static str = "  instances reincarnated:";
+                const ALREADY_REINCARNATED: &'static str =
+                    "  instances already reincarnated:";
+                const ERRORS: &'static str =
+                    "  instances which failed to be reincarnated:";
+                const WIDTH: usize = const_max_len(&[
+                    FOUND,
+                    REINCARNATED,
+                    ALREADY_REINCARNATED,
+                    ERRORS,
+                ]);
+                let n_restart_errors = restart_errors.len();
+                let n_restarted = instances_reincarnated.len();
+                let n_already_restarted = already_reincarnated.len();
+                println!("    {FOUND:<WIDTH$} {instances_found:>3}");
+                println!("    {REINCARNATED:<WIDTH$} {n_restarted:>3}");
+                println!(
+                    "    {ALREADY_REINCARNATED:<WIDTH$} {:>3}",
+                    n_already_restarted
+                );
+                println!("    {ERRORS:<WIDTH$} {n_restart_errors:>3}");
+
+                if let Some(e) = query_error {
+                    println!(
+                        "    an error occurred while searching for instances \
+                         to reincarnate:\n      {e}",
+                    );
+                }
+
+                if n_restart_errors > 0 {
+                    println!(
+                        "    errors occurred while restarting the following \
+                         instances:"
+                    );
+                    for (id, error) in restart_errors {
+                        println!("    > {id}: {error}");
+                    }
+                }
+
+                if n_restarted > 0 {
+                    println!("    the following instances have reincarnated:");
+                    for id in instances_reincarnated {
+                        println!("    > {id}")
+                    }
+                }
+
+                if n_already_restarted > 0 {
+                    println!(
+                        "    the following instances were reincarnated by another \
+                         Nexus\n    or a user-triggered start saga:"
+                    );
+                    for id in already_reincarnated {
+                        println!("    > {id}")
+                    }
+                }
+            }
+        };
     } else {
         println!(
             "warning: unknown background task: {:?} \
@@ -2354,3 +2429,16 @@ async fn cmd_nexus_sled_expunge_disk(
     eprintln!("expunged disk {}", args.physical_disk_id);
     Ok(())
 }
+
+const fn const_max_len(strs: &[&str]) -> usize {
+    let mut max = 0;
+    let mut i = 0;
+    while i < strs.len() {
+        let len = strs[i].len();
+        if len > max {
+            max = len;
+        }
+        i += 1;
+    }
+    max
+}

dev-tools/omdb/tests/env.out

@@ -86,6 +86,11 @@ task: "external_endpoints"
     on each one
 
 
+task: "instance_reincarnation"
+    schedules start sagas for failed instances that can be automatically
+    restarted
+
+
 task: "instance_updater"
     detects if instances require update sagas and schedules them
 
@@ -252,6 +257,11 @@ task: "external_endpoints"
     on each one
 
 
+task: "instance_reincarnation"
+    schedules start sagas for failed instances that can be automatically
+    restarted
+
+
 task: "instance_updater"
     detects if instances require update sagas and schedules them
 
@@ -405,6 +415,11 @@ task: "external_endpoints"
     on each one
 
 
+task: "instance_reincarnation"
+    schedules start sagas for failed instances that can be automatically
+    restarted
+
+
 task: "instance_updater"
     detects if instances require update sagas and schedules them
 

dev-tools/omdb/tests/successes.out

@@ -302,6 +302,11 @@ task: "external_endpoints"
     on each one
 
 
+task: "instance_reincarnation"
+    schedules start sagas for failed instances that can be automatically
+    restarted
+
+
 task: "instance_updater"
     detects if instances require update sagas and schedules them
 
@@ -518,6 +523,16 @@ task: "external_endpoints"
 
     TLS certificates: 0
 
+task: "instance_reincarnation"
+  configured period: every 1m
+  currently executing: no
+  last completed activation: <REDACTED ITERATIONS>, triggered by a periodic timer firing
+    started at <REDACTED     TIMESTAMP> (<REDACTED DURATION>s ago) and ran for <REDACTED DURATION>ms
+    instances eligible for reincarnation:          0
+      instances reincarnated:                      0
+      instances already reincarnated:              0
+      instances which failed to be reincarnated:   0
+
 task: "instance_updater"
   configured period: every <REDACTED_DURATION>s
   currently executing: no
@@ -945,6 +960,16 @@ task: "external_endpoints"
 
     TLS certificates: 0
 
+task: "instance_reincarnation"
+  configured period: every 1m
+  currently executing: no
+  last completed activation: <REDACTED ITERATIONS>, triggered by a periodic timer firing
+    started at <REDACTED     TIMESTAMP> (<REDACTED DURATION>s ago) and ran for <REDACTED DURATION>ms
+    instances eligible for reincarnation:          0
+      instances reincarnated:                      0
+      instances already reincarnated:              0
+      instances which failed to be reincarnated:   0
+
 task: "instance_updater"
   configured period: every <REDACTED_DURATION>s
   currently executing: no