plumb customer-configured DNS into instances

.github/buildomat/jobs/deploy.sh

@@ -2,7 +2,7 @@
 #:
 #: name = "helios / deploy"
 #: variety = "basic"
-#: target = "lab-2.0-opte-0.23"
+#: target = "lab-2.0-opte-0.25"
 #: output_rules = [
 #:  "%/var/svc/log/oxide-sled-agent:default.log*",
 #:  "%/pool/ext/*/crypt/zone/oxz_*/root/var/svc/log/oxide-*.log*",

Cargo.toml

@@ -249,15 +249,15 @@ omicron-sled-agent = { path = "sled-agent" }
 omicron-test-utils = { path = "test-utils" }
 omicron-zone-package = "0.8.3"
 oxide-client = { path = "clients/oxide-client" }
-oxide-vpc = { git = "https://github.com/oxidecomputer/opte", rev = "631c2017f19cafb1535f621e9e5aa9198ccad869", features = [ "api", "std" ] }
+oxide-vpc = { git = "https://github.com/oxidecomputer/opte", rev = "258a8b59902dd36fc7ee5425e6b1fb5fc80d4649", features = [ "api", "std" ] }
 once_cell = "1.18.0"
 openapi-lint = { git = "https://github.com/oxidecomputer/openapi-lint", branch = "main" }
 openapiv3 = "1.0"
 # must match samael's crate!
 openssl = "0.10"
 openssl-sys = "0.9"
 openssl-probe = "0.1.2"
-opte-ioctl = { git = "https://github.com/oxidecomputer/opte", rev = "631c2017f19cafb1535f621e9e5aa9198ccad869" }
+opte-ioctl = { git = "https://github.com/oxidecomputer/opte", rev = "258a8b59902dd36fc7ee5425e6b1fb5fc80d4649" }
 oso = "0.26"
 owo-colors = "3.5.0"
 oximeter = { path = "oximeter/oximeter" }

illumos-utils/src/opte/mod.rs

@@ -25,6 +25,7 @@ pub use port_manager::PortTicket;
 use ipnetwork::IpNetwork;
 use macaddr::MacAddr6;
 pub use oxide_vpc::api::BoundaryServices;
+pub use oxide_vpc::api::DhcpCfg;
 pub use oxide_vpc::api::Vni;
 use std::net::IpAddr;
 

illumos-utils/src/opte/params.rs

@@ -50,3 +50,26 @@ pub struct DeleteVirtualNetworkInterfaceHost {
     /// be deleted.
     pub vni: external::Vni,
 }
+
+/// DHCP configuration for a port
+///
+/// Not present here: Hostname (DHCPv4 option 12; used in DHCPv6 option 39); we
+/// use `InstanceRuntimeState::hostname` for this value.
+#[derive(Clone, Debug, Serialize, Deserialize, JsonSchema)]
+pub struct DhcpConfig {
+    /// DNS servers to send to the instance
+    ///
+    /// (DHCPv4 option 6; DHCPv6 option 23)
+    pub dns_servers: Vec<IpAddr>,
+
+    /// DNS zone this instance's hostname belongs to (e.g. the `project.example`
+    /// part of `instance1.project.example`)
+    ///
+    /// (DHCPv4 option 15; used in DHCPv6 option 39)
+    pub host_domain: Option<String>,
+
+    /// DNS search domains
+    ///
+    /// (DHCPv4 option 119; DHCPv6 option 24)
+    pub search_domains: Vec<String>,
+}

illumos-utils/src/opte/port_manager.rs

@@ -19,6 +19,7 @@ use omicron_common::api::internal::shared::NetworkInterface;
 use omicron_common::api::internal::shared::NetworkInterfaceKind;
 use omicron_common::api::internal::shared::SourceNatConfig;
 use oxide_vpc::api::AddRouterEntryReq;
+use oxide_vpc::api::DhcpCfg;
 use oxide_vpc::api::IpCfg;
 use oxide_vpc::api::IpCidr;
 use oxide_vpc::api::Ipv4Cfg;
@@ -100,6 +101,7 @@ impl PortManager {
         source_nat: Option<SourceNatConfig>,
         external_ips: &[IpAddr],
         firewall_rules: &[VpcFirewallRule],
+        dhcp_config: DhcpCfg,
     ) -> Result<(Port, PortTicket), Error> {
         let mac = *nic.mac;
         let vni = Vni::new(nic.vni).unwrap();
@@ -205,8 +207,6 @@ impl PortManager {
             vni,
             phys_ip: self.inner.underlay_ip.into(),
             boundary_services,
-            // TODO-completeness (#2153): Plumb domain search list
-            domain_list: vec![],
         };
 
         // Create the xde device.
@@ -227,11 +227,17 @@ impl PortManager {
             "Creating xde device";
             "port_name" => &port_name,
             "vpc_cfg" => ?&vpc_cfg,
+            "dhcp_config" => ?&dhcp_config,
         );
         #[cfg(target_os = "illumos")]
         let hdl = {
             let hdl = opte_ioctl::OpteHdl::open(opte_ioctl::OpteHdl::XDE_CTL)?;
-            hdl.create_xde(&port_name, vpc_cfg, /* passthru = */ false)?;
+            hdl.create_xde(
+                &port_name,
+                vpc_cfg,
+                dhcp_config,
+                /* passthru = */ false,
+            )?;
             hdl
         };
 

nexus/src/app/instance.rs

@@ -987,6 +987,12 @@ impl super::Nexus {
             source_nat,
             external_ips,
             firewall_rules,
+            dhcp_config: sled_agent_client::types::DhcpConfig {
+                dns_servers: self.external_dns_servers.clone(),
+                // TODO: finish designing instance DNS
+                host_domain: None,
+                search_domains: Vec::new(),
+            },
             disks: disk_reqs,
             cloud_init_bytes: Some(base64::Engine::encode(
                 &base64::engine::general_purpose::STANDARD,

nexus/src/app/mod.rs

@@ -26,7 +26,7 @@ use omicron_common::api::internal::shared::SwitchLocation;
 use omicron_common::nexus_config::RegionAllocationStrategy;
 use slog::Logger;
 use std::collections::HashMap;
-use std::net::Ipv6Addr;
+use std::net::{IpAddr, Ipv6Addr};
 use std::sync::Arc;
 use uuid::Uuid;
 
@@ -149,6 +149,12 @@ pub struct Nexus {
     /// DNS resolver Nexus uses to resolve an external host
     external_resolver: Arc<external_dns::Resolver>,
 
+    /// DNS servers used in `external_resolver`, used to provide DNS servers to
+    /// instances via DHCP
+    // TODO: This needs to be moved to the database.
+    // https://github.com/oxidecomputer/omicron/issues/3732
+    external_dns_servers: Vec<IpAddr>,
+
     /// Mapping of SwitchLocations to their respective Dendrite Clients
     dpd_clients: HashMap<SwitchLocation, Arc<dpd_client::Client>>,
 
@@ -327,6 +333,10 @@ impl Nexus {
             samael_max_issue_delay: std::sync::Mutex::new(None),
             internal_resolver: resolver,
             external_resolver,
+            external_dns_servers: config
+                .deployment
+                .external_dns_servers
+                .clone(),
             dpd_clients,
             background_tasks,
             default_region_allocation_strategy: config

openapi/sled-agent.json

@@ -1194,6 +1194,36 @@
           "vni"
         ]
       },
+      "DhcpConfig": {
+        "description": "DHCP configuration for a port\n\nNot present here: Hostname (DHCPv4 option 12; used in DHCPv6 option 39); we use `InstanceRuntimeState::hostname` for this value.",
+        "type": "object",
+        "properties": {
+          "dns_servers": {
+            "description": "DNS servers to send to the instance\n\n(DHCPv4 option 6; DHCPv6 option 23)",
+            "type": "array",
+            "items": {
+              "type": "string",
+              "format": "ip"
+            }
+          },
+          "host_domain": {
+            "nullable": true,
+            "description": "DNS zone this instance's hostname belongs to (e.g. the `project.example` part of `instance1.project.example`)\n\n(DHCPv4 option 15; used in DHCPv6 option 39)",
+            "type": "string"
+          },
+          "search_domains": {
+            "description": "DNS search domains\n\n(DHCPv4 option 119; DHCPv6 option 24)",
+            "type": "array",
+            "items": {
+              "type": "string"
+            }
+          }
+        },
+        "required": [
+          "dns_servers",
+          "search_domains"
+        ]
+      },
       "DiskEnsureBody": {
         "description": "Sent from to a sled agent to establish the runtime state of a Disk",
         "type": "object",
@@ -1697,6 +1727,9 @@
             "nullable": true,
             "type": "string"
           },
+          "dhcp_config": {
+            "$ref": "#/components/schemas/DhcpConfig"
+          },
           "disks": {
             "type": "array",
             "items": {
@@ -1731,6 +1764,7 @@
           }
         },
         "required": [
+          "dhcp_config",
           "disks",
           "external_ips",
           "firewall_rules",

sled-agent/Cargo.toml

@@ -44,7 +44,6 @@ macaddr.workspace = true
 nexus-client.workspace = true
 omicron-common.workspace = true
 once_cell.workspace = true
-oxide-vpc.workspace = true
 oximeter.workspace = true
 oximeter-producer.workspace = true
 percent-encoding.workspace = true

sled-agent/src/instance.rs

@@ -26,7 +26,7 @@ use chrono::Utc;
 use futures::lock::{Mutex, MutexGuard};
 use illumos_utils::dladm::Etherstub;
 use illumos_utils::link::VnicAllocator;
-use illumos_utils::opte::PortManager;
+use illumos_utils::opte::{DhcpCfg, PortManager};
 use illumos_utils::running_zone::{InstalledZone, RunningZone};
 use illumos_utils::svc::wait_for_service;
 use illumos_utils::zone::Zones;
@@ -91,6 +91,10 @@ pub enum Error {
     #[error(transparent)]
     Opte(#[from] illumos_utils::opte::Error),
 
+    /// Issued by `impl TryFrom<&[u8]> for oxide_vpc::api::DomainName`
+    #[error("Invalid hostname: {0}")]
+    InvalidHostname(&'static str),
+
     #[error("Error resolving DNS name: {0}")]
     ResolveError(#[from] internal_dns::resolver::ResolveError),
 
@@ -207,6 +211,7 @@ struct InstanceInner {
     source_nat: SourceNatConfig,
     external_ips: Vec<IpAddr>,
     firewall_rules: Vec<VpcFirewallRule>,
+    dhcp_config: DhcpCfg,
 
     // Disk related properties
     // TODO: replace `propolis_client::handmade::*` with properly-modeled local types
@@ -610,6 +615,37 @@ impl Instance {
             zone_bundler,
         } = services;
 
+        let mut dhcp_config = DhcpCfg {
+            hostname: Some(
+                hardware
+                    .properties
+                    .hostname
+                    .parse()
+                    .map_err(Error::InvalidHostname)?,
+            ),
+            host_domain: hardware
+                .dhcp_config
+                .host_domain
+                .map(|domain| domain.parse())
+                .transpose()
+                .map_err(Error::InvalidHostname)?,
+            domain_search_list: hardware
+                .dhcp_config
+                .search_domains
+                .into_iter()
+                .map(|domain| domain.parse())
+                .collect::<Result<_, _>>()
+                .map_err(Error::InvalidHostname)?,
+            dns4_servers: Vec::new(),
+            dns6_servers: Vec::new(),
+        };
+        for ip in hardware.dhcp_config.dns_servers {
+            match ip {
+                IpAddr::V4(ip) => dhcp_config.dns4_servers.push(ip.into()),
+                IpAddr::V6(ip) => dhcp_config.dns6_servers.push(ip.into()),
+            }
+        }
+
         let instance = InstanceInner {
             log: log.new(o!("instance_id" => id.to_string())),
             // NOTE: Mostly lies.
@@ -633,6 +669,7 @@ impl Instance {
             source_nat: hardware.source_nat,
             external_ips: hardware.external_ips,
             firewall_rules: hardware.firewall_rules,
+            dhcp_config,
             requested_disks: hardware.disks,
             cloud_init_bytes: hardware.cloud_init_bytes,
             state: InstanceStates::new(
@@ -852,6 +889,7 @@ impl Instance {
                 snat,
                 external_ips,
                 &inner.firewall_rules,
+                inner.dhcp_config.clone(),
             )?;
             opte_ports.push(port);
         }

sled-agent/src/params.rs

@@ -6,6 +6,7 @@ use crate::zone_bundle::PriorityOrder;
 pub use crate::zone_bundle::ZoneBundleCause;
 pub use crate::zone_bundle::ZoneBundleId;
 pub use crate::zone_bundle::ZoneBundleMetadata;
+pub use illumos_utils::opte::params::DhcpConfig;
 pub use illumos_utils::opte::params::VpcFirewallRule;
 pub use illumos_utils::opte::params::VpcFirewallRulesEnsureBody;
 use omicron_common::api::internal::nexus::{
@@ -68,6 +69,7 @@ pub struct InstanceHardware {
     /// provided to an instance to allow inbound connectivity.
     pub external_ips: Vec<IpAddr>,
     pub firewall_rules: Vec<VpcFirewallRule>,
+    pub dhcp_config: DhcpConfig,
     // TODO: replace `propolis_client::handmade::*` with locally-modeled request type
     pub disks: Vec<propolis_client::handmade::api::DiskRequest>,
     pub cloud_init_bytes: Option<String>,

sled-agent/src/services.rs

@@ -52,7 +52,7 @@ use illumos_utils::dladm::{
     Dladm, Etherstub, EtherstubVnic, GetSimnetError, PhysicalLink,
 };
 use illumos_utils::link::{Link, VnicAllocator};
-use illumos_utils::opte::{Port, PortManager, PortTicket};
+use illumos_utils::opte::{DhcpCfg, Port, PortManager, PortTicket};
 use illumos_utils::running_zone::{
     InstalledZone, RunCommandError, RunningZone,
 };
@@ -862,11 +862,11 @@ impl ServiceManager {
             // config allows outbound access which is enough for
             // Boundary NTP which needs to come up before Nexus.
             let port = port_manager
-                .create_port(nic, snat, external_ips, &[])
+                .create_port(nic, snat, external_ips, &[], DhcpCfg::default())
                 .map_err(|err| Error::ServicePortCreation {
-                service: svc.details.to_string(),
-                err: Box::new(err),
-            })?;
+                    service: svc.details.to_string(),
+                    err: Box::new(err),
+                })?;
 
             // We also need to update the switch with the NAT mappings
             let (target_ip, first_port, last_port) = match snat {