[nexus] Reincarnate Failed instances

common/src/api/external/mod.rs

@@ -1158,6 +1158,12 @@ impl From<&InstanceCpuCount> for i64 {
 pub struct InstanceRuntimeState {
     pub run_state: InstanceState,
     pub time_run_state_updated: DateTime<Utc>,
+    /// The timestamp of the most recent time this instance was automatically
+    /// restarted by the control plane.
+    ///
+    /// If this is not present, then this instance has not been automatically
+    /// restarted.
+    pub time_last_auto_restarted: Option<DateTime<Utc>>,
 }
 
 /// View of an Instance
@@ -1179,6 +1185,52 @@ pub struct Instance {
 
     #[serde(flatten)]
     pub runtime: InstanceRuntimeState,
+
+    #[serde(flatten)]
+    pub auto_restart_status: InstanceAutoRestartStatus,
+}
+
+/// Status of control-plane driven automatic failure recovery for this instance.
+#[derive(Clone, Debug, Deserialize, Serialize, JsonSchema)]
+pub struct InstanceAutoRestartStatus {
+    /// `true` if this instance's auto-restart policy will permit the control
+    /// plane to automatically restart it if it enters the `Failed` state.
+    //
+    // Rename this field, as the struct is `#[serde(flatten)]`ed into the
+    // `Instance` type, and we would like the field to be prefixed with
+    // `auto_restart`.
+    #[serde(rename = "auto_restart_enabled")]
+    pub enabled: bool,
+
+    /// The time at which the auto-restart cooldown period for this instance
+    /// completes, permitting it to be automatically restarted again. If the
+    /// instance enters the `Failed` state, it will not be restarted until after
+    /// this time.
+    ///
+    /// If this is not present, then either the instance has never been
+    /// automatically restarted, or the cooldown period has already expired,
+    /// allowing the instance to be restarted immediately if it fails.
+    //
+    // Rename this field, as the struct is `#[serde(flatten)]`ed into the
+    // `Instance` type, and we would like the field to be prefixed with
+    // `auto_restart`.
+    #[serde(rename = "auto_restart_cooldown_expiration")]
+    pub cooldown_expiration: Option<DateTime<Utc>>,
+}
+
+/// A policy determining when an instance should be automatically restarted by
+/// the control plane.
+#[derive(Copy, Clone, Debug, Deserialize, Serialize, JsonSchema)]
+#[serde(rename_all = "snake_case")]
+pub enum InstanceAutoRestartPolicy {
+    /// The instance should not be automatically restarted by the control plane
+    /// if it fails.
+    Never,
+    /// If this instance is running and unexpectedly fails (e.g. due to a host
+    /// software crash or unexpected host reboot), the control plane will make a
+    /// best-effort attempt to restart it. The control plane may choose not to
+    /// restart the instance to preserve the overall availability of the system.
+    BestEffort,
 }
 
 // DISKS

dev-tools/omdb/src/bin/omdb/nexus.rs

@@ -35,6 +35,7 @@ use nexus_db_queries::db::lookup::LookupPath;
 use nexus_saga_recovery::LastPass;
 use nexus_types::deployment::Blueprint;
 use nexus_types::internal_api::background::AbandonedVmmReaperStatus;
+use nexus_types::internal_api::background::InstanceReincarnationStatus;
 use nexus_types::internal_api::background::InstanceUpdaterStatus;
 use nexus_types::internal_api::background::LookupRegionPortStatus;
 use nexus_types::internal_api::background::RegionReplacementDriverStatus;
@@ -1780,6 +1781,80 @@ fn print_task_details(bgtask: &BackgroundTask, details: &serde_json::Value) {
                 }
             }
         }
+    } else if name == "instance_reincarnation" {
+        match serde_json::from_value::<InstanceReincarnationStatus>(
+            details.clone(),
+        ) {
+            Err(error) => eprintln!(
+                "warning: failed to interpret task details: {:?}: {:?}",
+                error, details
+            ),
+            Ok(InstanceReincarnationStatus {
+                instances_found,
+                instances_reincarnated,
+                changed_state,
+                query_error,
+                restart_errors,
+            }) => {
+                const FOUND: &'static str =
+                    "instances eligible for reincarnation:";
+                const REINCARNATED: &'static str = "  instances reincarnated:";
+                const CHANGED_STATE: &'static str =
+                    "  instances which changed state before they could be reincarnated:";
+                const ERRORS: &'static str =
+                    "  instances which failed to be reincarnated:";
+                const COOLDOWN_PERIOD: &'static str =
+                    "default cooldown period:";
+                const WIDTH: usize = const_max_len(&[
+                    FOUND,
+                    REINCARNATED,
+                    CHANGED_STATE,
+                    ERRORS,
+                    COOLDOWN_PERIOD,
+                ]);
+                let n_restart_errors = restart_errors.len();
+                let n_restarted = instances_reincarnated.len();
+                let n_changed_state = changed_state.len();
+                println!("    {FOUND:<WIDTH$} {instances_found:>3}");
+                println!("    {REINCARNATED:<WIDTH$} {n_restarted:>3}");
+                println!("    {CHANGED_STATE:<WIDTH$} {n_changed_state:>3}",);
+                println!("    {ERRORS:<WIDTH$} {n_restart_errors:>3}");
+
+                if let Some(e) = query_error {
+                    println!(
+                        "    an error occurred while searching for instances \
+                         to reincarnate:\n      {e}",
+                    );
+                }
+
+                if n_restart_errors > 0 {
+                    println!(
+                        "    errors occurred while restarting the following \
+                         instances:"
+                    );
+                    for (id, error) in restart_errors {
+                        println!("    > {id}: {error}");
+                    }
+                }
+
+                if n_restarted > 0 {
+                    println!("    the following instances have reincarnated:");
+                    for id in instances_reincarnated {
+                        println!("    > {id}")
+                    }
+                }
+
+                if n_changed_state > 0 {
+                    println!(
+                        "    the following instances states changed before \
+                         they could be reincarnated:"
+                    );
+                    for id in changed_state {
+                        println!("    > {id}")
+                    }
+                }
+            }
+        };
     } else {
         println!(
             "warning: unknown background task: {:?} \

dev-tools/omdb/tests/env.out

@@ -86,6 +86,11 @@ task: "external_endpoints"
     on each one
 
 
+task: "instance_reincarnation"
+    schedules start sagas for failed instances that can be automatically
+    restarted
+
+
 task: "instance_updater"
     detects if instances require update sagas and schedules them
 
@@ -252,6 +257,11 @@ task: "external_endpoints"
     on each one
 
 
+task: "instance_reincarnation"
+    schedules start sagas for failed instances that can be automatically
+    restarted
+
+
 task: "instance_updater"
     detects if instances require update sagas and schedules them
 
@@ -405,6 +415,11 @@ task: "external_endpoints"
     on each one
 
 
+task: "instance_reincarnation"
+    schedules start sagas for failed instances that can be automatically
+    restarted
+
+
 task: "instance_updater"
     detects if instances require update sagas and schedules them
 

dev-tools/omdb/tests/successes.out

@@ -302,6 +302,11 @@ task: "external_endpoints"
     on each one
 
 
+task: "instance_reincarnation"
+    schedules start sagas for failed instances that can be automatically
+    restarted
+
+
 task: "instance_updater"
     detects if instances require update sagas and schedules them
 
@@ -518,6 +523,16 @@ task: "external_endpoints"
 
     TLS certificates: 0
 
+task: "instance_reincarnation"
+  configured period: every 1m
+  currently executing: no
+  last completed activation: <REDACTED ITERATIONS>, triggered by a periodic timer firing
+    started at <REDACTED_TIMESTAMP> (<REDACTED DURATION>s ago) and ran for <REDACTED DURATION>ms
+    instances eligible for reincarnation:                                0
+      instances reincarnated:                                            0
+      instances which changed state before they could be reincarnated:   0
+      instances which failed to be reincarnated:                         0
+
 task: "instance_updater"
   configured period: every <REDACTED_DURATION>s
   currently executing: no
@@ -948,6 +963,16 @@ task: "external_endpoints"
 
     TLS certificates: 0
 
+task: "instance_reincarnation"
+  configured period: every 1m
+  currently executing: no
+  last completed activation: <REDACTED ITERATIONS>, triggered by a periodic timer firing
+    started at <REDACTED_TIMESTAMP> (<REDACTED DURATION>s ago) and ran for <REDACTED DURATION>ms
+    instances eligible for reincarnation:                                0
+      instances reincarnated:                                            0
+      instances which changed state before they could be reincarnated:   0
+      instances which failed to be reincarnated:                         0
+
 task: "instance_updater"
   configured period: every <REDACTED_DURATION>s
   currently executing: no

end-to-end-tests/src/instance_launch.rs

@@ -76,6 +76,7 @@ async fn instance_launch() -> Result<()> {
                 ssh_key_name.clone(),
             )]),
             start: true,
+            auto_restart_policy: Default::default(),
         })
         .send()
         .await?;

nexus-config/src/nexus_config.rs

@@ -382,6 +382,8 @@ pub struct BackgroundTaskConfig {
     pub instance_watcher: InstanceWatcherConfig,
     /// configuration for instance updater task
     pub instance_updater: InstanceUpdaterConfig,
+    /// configuration for instance reincarnation task
+    pub instance_reincarnation: InstanceReincarnationConfig,
     /// configuration for service VPC firewall propagation task
     pub service_firewall_propagation: ServiceFirewallPropagationConfig,
     /// configuration for v2p mapping propagation task
@@ -590,6 +592,23 @@ pub struct InstanceUpdaterConfig {
     pub disable: bool,
 }
 
+#[serde_as]
+#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)]
+pub struct InstanceReincarnationConfig {
+    /// period (in seconds) for periodic activations of this background task
+    #[serde_as(as = "DurationSeconds<u64>")]
+    pub period_secs: Duration,
+
+    /// disable background checks for instances in need of updates.
+    ///
+    /// This is an emergency lever for support / operations. It should only be
+    /// necessary if something has gone extremely wrong.
+    ///
+    /// Default: Off
+    #[serde(default)]
+    pub disable: bool,
+}
+
 #[serde_as]
 #[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)]
 pub struct ServiceFirewallPropagationConfig {
@@ -912,6 +931,7 @@ mod test {
             instance_watcher.period_secs = 30
             instance_updater.period_secs = 30
             instance_updater.disable = false
+            instance_reincarnation.period_secs = 67
             service_firewall_propagation.period_secs = 300
             v2p_mapping_propagation.period_secs = 30
             abandoned_vmm_reaper.period_secs = 60
@@ -1067,6 +1087,10 @@ mod test {
                             period_secs: Duration::from_secs(30),
                             disable: false,
                         },
+                        instance_reincarnation: InstanceReincarnationConfig {
+                            period_secs: Duration::from_secs(67),
+                            disable: false,
+                        },
                         service_firewall_propagation:
                             ServiceFirewallPropagationConfig {
                                 period_secs: Duration::from_secs(300),
@@ -1170,6 +1194,7 @@ mod test {
             region_replacement_driver.period_secs = 30
             instance_watcher.period_secs = 30
             instance_updater.period_secs = 30
+            instance_reincarnation.period_secs = 67
             service_firewall_propagation.period_secs = 300
             v2p_mapping_propagation.period_secs = 30
             abandoned_vmm_reaper.period_secs = 60