[sled-agent] Add ability to allocate guest-visible NICs (#186)

omicron-common/Cargo.toml

@@ -12,7 +12,7 @@ futures = "0.3.15"
 http = "0.2.0"
 hyper = "0.14"
 libc = "0.2.98"
-propolis-server = { git = "https://github.com/oxidecomputer/propolis", rev = "b6da043d" }
+propolis-server = { git = "https://github.com/oxidecomputer/propolis", rev = "bc0661e" }
 postgres-protocol = "0.6.1"
 rayon = "1.5"
 reqwest = { version = "0.11", default-features = false, features = ["rustls-tls"] }

omicron-common/src/api/internal/sled_agent.rs

@@ -1,6 +1,6 @@
 //! APIs exposed by Sled Agent.
 
-use crate::api::internal;
+use crate::api::{external, internal};
 use schemars::JsonSchema;
 use serde::{Deserialize, Serialize};
 use std::fmt::{Debug, Display, Formatter, Result as FormatResult};
@@ -39,15 +39,19 @@ impl DiskStateRequested {
     }
 }
 
-/// Runtime state of an instance.
-pub type InstanceRuntimeState = internal::nexus::InstanceRuntimeState;
+/// Describes the instance hardware.
+#[derive(Clone, Debug, Serialize, Deserialize, JsonSchema)]
+pub struct InstanceHardware {
+    pub runtime: internal::nexus::InstanceRuntimeState,
+    pub nics: Vec<external::NetworkInterface>,
+}
 
 /// Sent to a sled agent to establish the runtime state of an Instance
 #[derive(Serialize, Deserialize, JsonSchema)]
 pub struct InstanceEnsureBody {
     /// Last runtime state of the Instance known to Nexus (used if the agent
     /// has never seen this Instance before).
-    pub initial_runtime: InstanceRuntimeState,
+    pub initial: InstanceHardware,
     /// requested runtime state of the Instance
     pub target: InstanceRuntimeStateRequested,
 }

omicron-common/src/sled_agent_client.rs

@@ -5,12 +5,14 @@
  * generated by the server.
  */
 
-use crate::api;
 use crate::api::external::Error;
 use crate::api::internal::nexus::DiskRuntimeState;
+use crate::api::internal::nexus::InstanceRuntimeState;
 use crate::api::internal::sled_agent::DiskEnsureBody;
 use crate::api::internal::sled_agent::DiskStateRequested;
 use crate::api::internal::sled_agent::InstanceEnsureBody;
+use crate::api::internal::sled_agent::InstanceHardware;
+use crate::api::internal::sled_agent::InstanceRuntimeStateRequested;
 use crate::http_client::HttpClient;
 use async_trait::async_trait;
 use http::Method;
@@ -53,24 +55,21 @@ impl Client {
     pub async fn instance_ensure(
         self: &Arc<Self>,
         instance_id: Uuid,
-        initial_runtime: api::internal::sled_agent::InstanceRuntimeState,
-        target: api::internal::sled_agent::InstanceRuntimeStateRequested,
-    ) -> Result<api::internal::sled_agent::InstanceRuntimeState, Error> {
+        initial: InstanceHardware,
+        target: InstanceRuntimeStateRequested,
+    ) -> Result<InstanceRuntimeState, Error> {
         let path = format!("/instances/{}", instance_id);
         let body = Body::from(
-            serde_json::to_string(&InstanceEnsureBody {
-                initial_runtime,
-                target,
-            })
-            .unwrap(),
+            serde_json::to_string(&InstanceEnsureBody { initial, target })
+                .unwrap(),
         );
         let mut response =
             self.client.request(Method::PUT, path.as_str(), body).await?;
         /* TODO-robustness handle 300-level? */
         assert!(response.status().is_success());
         let value = self
             .client
-            .read_json::<api::internal::sled_agent::InstanceRuntimeState>(
+            .read_json::<InstanceRuntimeState>(
                 &self.client.error_message_base(&Method::PUT, path.as_str()),
                 &mut response,
             )

omicron-nexus/Cargo.toml

@@ -16,6 +16,7 @@ hyper = "0.14"
 ipnetwork = "0.18"
 lazy_static = "1.4.0"
 libc = "0.2.98"
+macaddr = { version = "1.0.1", features = [ "serde_std" ]}
 newtype_derive = "0.1.6"
 serde_json = "1.0"
 serde_with = "1.9.4"

omicron-nexus/src/db/model.rs

@@ -382,7 +382,7 @@ impl From<internal::nexus::InstanceRuntimeState> for InstanceRuntimeState {
 /// Conversion to the internal API type.
 impl Into<internal::nexus::InstanceRuntimeState> for InstanceRuntimeState {
     fn into(self) -> internal::nexus::InstanceRuntimeState {
-        internal::sled_agent::InstanceRuntimeState {
+        internal::nexus::InstanceRuntimeState {
             run_state: *self.state.state(),
             sled_uuid: self.sled_uuid,
             ncpus: self.ncpus,

omicron-nexus/src/nexus.rs

@@ -39,6 +39,7 @@ use omicron_common::api::internal::nexus::DiskRuntimeState;
 use omicron_common::api::internal::nexus::OximeterInfo;
 use omicron_common::api::internal::nexus::ProducerEndpoint;
 use omicron_common::api::internal::sled_agent::DiskStateRequested;
+use omicron_common::api::internal::sled_agent::InstanceHardware;
 use omicron_common::api::internal::sled_agent::InstanceRuntimeStateRequested;
 use omicron_common::api::internal::sled_agent::InstanceStateRequested;
 use omicron_common::bail_unless;
@@ -762,8 +763,17 @@ impl Nexus {
          * not the newest one, that's fine.  That might just mean the sled agent
          * beat us to it.
          */
+
+        // TODO: Populate this with an appropriate NIC.
+        // See also: sic_create_instance_record in sagas.rs for a similar
+        // construction.
+        let instance_hardware = InstanceHardware {
+            runtime: instance.runtime().into(),
+            nics: vec![],
+        };
+
         let new_runtime = sa
-            .instance_ensure(instance.id, instance.runtime().into(), requested)
+            .instance_ensure(instance.id, instance_hardware, requested)
             .await?;
 
         self.db_datastore

omicron-nexus/src/sagas.rs

@@ -16,6 +16,7 @@ use omicron_common::api::external::Generation;
 use omicron_common::api::external::InstanceCreateParams;
 use omicron_common::api::external::InstanceState;
 use omicron_common::api::internal::nexus::InstanceRuntimeState;
+use omicron_common::api::internal::sled_agent::InstanceHardware;
 use omicron_common::api::internal::sled_agent::InstanceRuntimeStateRequested;
 use omicron_common::api::internal::sled_agent::InstanceStateRequested;
 use serde::Deserialize;
@@ -125,7 +126,7 @@ async fn sic_alloc_server(
 
 async fn sic_create_instance_record(
     sagactx: ActionContext<SagaInstanceCreate>,
-) -> Result<InstanceRuntimeState, ActionError> {
+) -> Result<InstanceHardware, ActionError> {
     let osagactx = sagactx.user_data();
     let params = sagactx.saga_params();
     let sled_uuid = sagactx.lookup::<Uuid>("server_id");
@@ -151,7 +152,10 @@ async fn sic_create_instance_record(
         )
         .await
         .map_err(ActionError::action_failed)?;
-    Ok(instance.runtime().into())
+
+    // TODO: Populate this with an appropriate NIC.
+    // See also: instance_set_runtime in nexus.rs for a similar construction.
+    Ok(InstanceHardware { runtime: instance.runtime().into(), nics: vec![] })
 }
 
 async fn sic_instance_ensure(
@@ -167,7 +171,7 @@ async fn sic_instance_ensure(
     let instance_id = sagactx.lookup::<Uuid>("instance_id")?;
     let sled_uuid = sagactx.lookup::<Uuid>("server_id")?;
     let initial_runtime =
-        sagactx.lookup::<InstanceRuntimeState>("initial_runtime")?;
+        sagactx.lookup::<InstanceHardware>("initial_runtime")?;
     let sa = osagactx
         .sled_client(&sled_uuid)
         .await

omicron-sled-agent/Cargo.toml

@@ -13,7 +13,7 @@ http = "0.2.0"
 hyper = "0.14"
 ipnet = "2.3"
 omicron-common = { path = "../omicron-common" }
-propolis-client = { git = "https://github.com/oxidecomputer/propolis", rev = "3d58a6398" }
+propolis-client = { git = "https://github.com/oxidecomputer/propolis", rev = "bc0661e" }
 schemars = { version = "0.8", features = [ "chrono", "uuid" ] }
 serde = { version = "1.0", features = [ "derive" ] }
 serde_json = "1.0"

omicron-sled-agent/src/bin/sled-agent.rs

@@ -5,6 +5,7 @@ use dropshot::ConfigLogging;
 use dropshot::ConfigLoggingLevel;
 use omicron_common::cmd::fatal;
 use omicron_common::cmd::CmdError;
+use omicron_sled_agent::common::vlan::VlanID;
 use omicron_sled_agent::config::Config;
 use omicron_sled_agent::server::{run_openapi, run_server};
 use std::net::SocketAddr;
@@ -29,6 +30,9 @@ enum Args {
 
         #[structopt(name = "NEXUS_IP:PORT", parse(try_from_str))]
         nexus_addr: SocketAddr,
+
+        #[structopt(long = "vlan")]
+        vlan: Option<VlanID>,
     },
 }
 
@@ -46,7 +50,7 @@ async fn do_run() -> Result<(), CmdError> {
 
     match args {
         Args::OpenApi => run_openapi().map_err(CmdError::Failure),
-        Args::Run { uuid, sled_agent_addr, nexus_addr } => {
+        Args::Run { uuid, sled_agent_addr, nexus_addr, vlan } => {
             let config = Config {
                 id: uuid,
                 nexus_address: nexus_addr,
@@ -57,6 +61,7 @@ async fn do_run() -> Result<(), CmdError> {
                 log: ConfigLogging::StderrTerminal {
                     level: ConfigLoggingLevel::Info,
                 },
+                vlan,
             };
             run_server(&config).await.map_err(CmdError::Failure)
         }

omicron-sled-agent/src/common/mod.rs

@@ -2,3 +2,4 @@
 
 pub mod disk;
 pub mod instance;
+pub mod vlan;

omicron-sled-agent/src/common/vlan.rs

@@ -0,0 +1,41 @@
+//! VLAN ID wrapper.
+
+use omicron_common::api::external::Error;
+use std::fmt;
+use std::str::FromStr;
+
+/// The maximum VLAN value (inclusive), as specified by IEEE 802.1Q.
+pub const VLAN_MAX: u16 = 4094;
+
+/// Wrapper around a VLAN ID, ensuring it is valid.
+#[derive(Debug, Clone, Copy)]
+pub struct VlanID(u16);
+
+impl VlanID {
+    /// Creates a new VLAN ID, returning an error if it is out of range.
+    pub fn new(id: u16) -> Result<Self, Error> {
+        if VLAN_MAX < id {
+            return Err(Error::InvalidValue {
+                label: id.to_string(),
+                message: "Invalid VLAN value".to_string(),
+            });
+        }
+        Ok(Self(id))
+    }
+}
+
+impl fmt::Display for VlanID {
+    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+        write!(f, "{}", self.0.to_string())
+    }
+}
+
+impl FromStr for VlanID {
+    type Err = Error;
+    fn from_str(s: &str) -> Result<Self, Self::Err> {
+        Self::new(s.parse().map_err(|e| Error::InvalidValue {
+            label: s.to_string(),
+            message: format!("{}", e),
+        })?)
+    }
+}

omicron-sled-agent/src/config.rs

@@ -1,14 +1,13 @@
 //! Interfaces for working with sled agent configuration
 
+use crate::common::vlan::VlanID;
 use dropshot::ConfigDropshot;
 use dropshot::ConfigLogging;
-use serde::Deserialize;
-use serde::Serialize;
 use std::net::SocketAddr;
 use uuid::Uuid;
 
 /// Configuration for a sled agent
-#[derive(Clone, Debug, Deserialize, PartialEq, Serialize)]
+#[derive(Clone, Debug)]
 pub struct Config {
     /// Unique id for the sled
     pub id: Uuid,
@@ -18,4 +17,6 @@ pub struct Config {
     pub dropshot: ConfigDropshot,
     /// Configuration for the sled agent debug log
     pub log: ConfigLogging,
+    /// Optional VLAN ID to be used for tagging guest VNICs.
+    pub vlan: Option<VlanID>,
 }

omicron-sled-agent/src/http_entrypoints.rs

@@ -54,12 +54,8 @@ async fn instance_put(
     let instance_id = path_params.into_inner().instance_id;
     let body_args = body.into_inner();
     Ok(HttpResponseOk(
-        sa.instance_ensure(
-            instance_id,
-            body_args.initial_runtime.clone(),
-            body_args.target.clone(),
-        )
-        .await?,
+        sa.instance_ensure(instance_id, body_args.initial, body_args.target)
+            .await?,
     ))
 }
 

omicron-sled-agent/src/illumos/dladm.rs

@@ -1,23 +1,28 @@
 //! Utilities for poking at data links.
 
+use crate::common::vlan::VlanID;
 use crate::illumos::{execute, PFEXEC};
 use omicron_common::api::external::Error;
+use omicron_common::api::external::MacAddr;
 
 pub const VNIC_PREFIX: &str = "vnic_propolis";
+pub const DLADM: &str = "/usr/sbin/dladm";
 
-const DLADM: &str = "/usr/sbin/dladm";
+/// The name of a physical datalink.
+#[derive(Debug, Clone)]
+pub struct PhysicalLink(pub String);
 
 /// Wraps commands for interacting with data links.
 pub struct Dladm {}
 
 #[cfg_attr(test, mockall::automock, allow(dead_code))]
 impl Dladm {
     /// Returns the name of the first observed physical data link.
-    pub fn find_physical() -> Result<String, Error> {
+    pub fn find_physical() -> Result<PhysicalLink, Error> {
         let mut command = std::process::Command::new(PFEXEC);
         let cmd = command.args(&[DLADM, "show-phys", "-p", "-o", "LINK"]);
         let output = execute(cmd)?;
-        Ok(String::from_utf8(output.stdout)
+        let name = String::from_utf8(output.stdout)
             .map_err(|e| Error::InternalError {
                 message: format!("Cannot parse dladm output as UTF-8: {}", e),
             })?
@@ -30,14 +35,44 @@ impl Dladm {
             .ok_or_else(|| Error::InternalError {
                 message: "No physical devices found".to_string(),
             })?
-            .to_string())
+            .to_string();
+        Ok(PhysicalLink(name))
     }
 
     /// Creates a new VNIC atop a physical device.
-    pub fn create_vnic(physical: &str, vnic_name: &str) -> Result<(), Error> {
+    ///
+    /// * `physical`: The physical link on top of which a device will be
+    /// created.
+    /// * `vnic_name`: Exact name of the VNIC to be created.
+    /// * `mac`: An optional unicast MAC address for the newly created NIC.
+    /// * `vlan`: An optional VLAN ID for VLAN tagging.
+    pub fn create_vnic(
+        physical: &PhysicalLink,
+        vnic_name: &str,
+        mac: Option<MacAddr>,
+        vlan: Option<VlanID>,
+    ) -> Result<(), Error> {
         let mut command = std::process::Command::new(PFEXEC);
-        let cmd =
-            command.args(&[DLADM, "create-vnic", "-l", physical, vnic_name]);
+        let mut args = vec![
+            DLADM.to_string(),
+            "create-vnic".to_string(),
+            "-t".to_string(),
+            "-l".to_string(),
+            physical.0.to_string(),
+        ];
+
+        if let Some(mac) = mac {
+            args.push("-m".to_string());
+            args.push(mac.0.to_string());
+        }
+
+        if let Some(vlan) = vlan {
+            args.push("-v".to_string());
+            args.push(vlan.to_string());
+        }
+
+        args.push(vnic_name.to_string());
+        let cmd = command.args(&args);
         execute(cmd)?;
         Ok(())
     }