Support for omicron internet gateway model (#588)

The Omicron internet gateway model associates IP pools, and subsequently
addresses from those IP pools, with internet gateways. When packets are
directed at an internet gateway through a VPC route, they should take on
a source address from the IP pool that is tied to the target internet
gateway. This trivially happens when there is only one internet gateway
and an instance only has an address from one IP pool. But when there are
multiple IP pools in play, particularly with floating IPs from different
pools, a decision needs to be made about which one to use. That question
is answered by the IP-pool/internet-gateway association in combination
with VPC routes that direct packets at a particular internet gateway.

---------

Co-authored-by: Kyle Simpson <[email protected]>

.github/buildomat/jobs/opteadm.sh

@@ -11,6 +11,10 @@
 #:   "=/work/release/opteadm.release.sha256",
 #: ]
 #:
+#: [[publish]]
+#: series = "release"
+#: name = "opteadm"
+#: from_output = "/work/release/opteadm"
 
 set -o errexit
 set -o pipefail

.github/buildomat/jobs/xde.sh

@@ -21,7 +21,7 @@
 #: series = "module"
 #: name = "xde"
 #: from_output = "/work/release/xde"
-#
+#:
 #: [[publish]]
 #: series = "module"
 #: name = "xde.sha256"

Cargo.toml

@@ -72,6 +72,7 @@ syn = "2"
 tabwriter = { version = "1", features = ["ansi_formatting"] }
 thiserror = "1.0"
 toml = "0.8"
+uuid = { version = "1.0", default-features = false, features = ["serde"]}
 usdt = "0.5"
 version_check = "0.9"
 zerocopy = { version = "0.7", features = ["derive"] }

bench/src/packet.rs

@@ -294,7 +294,7 @@ impl BenchPacketInstance for UlpProcessInstance {
         router::add_entry(
             &g1.port,
             IpCidr::Ip4("0.0.0.0/0".parse().unwrap()),
-            RouterTarget::InternetGateway,
+            RouterTarget::InternetGateway(None),
             RouterClass::System,
         )
         .unwrap();
@@ -303,7 +303,7 @@ impl BenchPacketInstance for UlpProcessInstance {
         router::add_entry(
             &g1.port,
             IpCidr::Ip6("::/0".parse().unwrap()),
-            RouterTarget::InternetGateway,
+            RouterTarget::InternetGateway(None),
             RouterClass::System,
         )
         .unwrap();

bin/opteadm/src/bin/opteadm.rs

@@ -782,6 +782,10 @@ fn main() -> anyhow::Result<()> {
                     port_name: port,
                     external_ips_v4: Some(cfg),
                     external_ips_v6: None,
+                    // TODO: It'd be nice to have this user-specifiable via
+                    //       opteadm, but for now it's a purely control-plane
+                    //       concept.
+                    inet_gw_map: None,
                 };
                 hdl.set_external_ips(&req)?;
             } else if let Ok(cfg) =
@@ -791,6 +795,8 @@ fn main() -> anyhow::Result<()> {
                     port_name: port,
                     external_ips_v6: Some(cfg),
                     external_ips_v4: None,
+                    // TODO: As above.
+                    inet_gw_map: None,
                 };
                 hdl.set_external_ips(&req)?;
             } else {

lib/opte-test-utils/src/lib.rs

@@ -102,6 +102,9 @@ pub const BS_IP_ADDR: Ipv6Addr =
 const UFT_LIMIT: Option<NonZeroU32> = NonZeroU32::new(16);
 const TCP_LIMIT: Option<NonZeroU32> = NonZeroU32::new(16);
 
+pub const EXT_IP4: &str = "10.77.77.13";
+pub const EXT_IP6: &str = "fd00:100::1";
+
 pub fn ox_vpc_mac(id: [u8; 3]) -> MacAddr {
     MacAddr::from([0xA8, 0x40, 0x25, 0xF0 | id[0], id[1], id[2]])
 }
@@ -132,7 +135,7 @@ pub fn g1_cfg() -> VpcCfg {
             gateway_ip: "172.30.0.1".parse().unwrap(),
             external_ips: ExternalIpCfg {
                 snat: Some(SNat4Cfg {
-                    external_ip: "10.77.77.13".parse().unwrap(),
+                    external_ip: EXT_IP4.parse().unwrap(),
                     ports: 1025..=4096,
                 }),
                 ephemeral_ip: None,
@@ -372,7 +375,8 @@ pub fn oxide_net_setup2(
         "set:firewall.rules.out=0",
         // * Outbound IPv4 SNAT
         // * Outbound IPv6 SNAT
-        "set:nat.rules.out=2",
+        // * Drop uncaught InetGw packets.
+        "set:nat.rules.out=3",
     ];
 
     [

lib/oxide-vpc/Cargo.toml

@@ -33,12 +33,13 @@ usdt = ["opte/usdt"]
 illumos-sys-hdrs.workspace = true
 opte.workspace = true
 
+cfg-if.workspace = true
+poptrie.workspace = true
 serde.workspace = true
 smoltcp.workspace = true
 tabwriter = { workspace = true, optional = true }
+uuid.workspace = true
 zerocopy.workspace = true
-poptrie.workspace = true
-cfg-if.workspace = true
 
 [dev-dependencies]
 ctor.workspace = true

lib/oxide-vpc/src/api.rs

@@ -4,6 +4,7 @@
 
 // Copyright 2024 Oxide Computer Company
 
+use alloc::collections::BTreeMap;
 use alloc::collections::BTreeSet;
 use alloc::string::String;
 use alloc::string::ToString;
@@ -16,6 +17,7 @@ use illumos_sys_hdrs::datalink_id_t;
 pub use opte::api::*;
 use serde::Deserialize;
 use serde::Serialize;
+use uuid::Uuid;
 
 /// This is the MAC address that OPTE uses to act as the virtual gateway.
 pub const GW_MAC_ADDR: MacAddr =
@@ -347,7 +349,8 @@ impl From<PhysNet> for GuestPhysAddr {
 /// * InternetGateway: Packets matching this entry are forwarded to
 /// the internet. In the case of the Oxide Network the IG is not an
 /// actual destination, but rather a configuration that determines how
-/// we should NAT the flow.
+/// we should NAT the flow. The address in the gateway is the source
+/// address that is to be used.
 ///
 /// * Ip: Packets matching this entry are forwarded to the specified IP.
 ///
@@ -362,7 +365,7 @@ impl From<PhysNet> for GuestPhysAddr {
 #[derive(Clone, Debug, Copy, Deserialize, Serialize)]
 pub enum RouterTarget {
     Drop,
-    InternetGateway,
+    InternetGateway(Option<Uuid>),
     Ip(IpAddr),
     VpcSubnet(IpCidr),
 }
@@ -374,7 +377,7 @@ impl FromStr for RouterTarget {
     fn from_str(s: &str) -> Result<Self, Self::Err> {
         match s.to_ascii_lowercase().as_str() {
             "drop" => Ok(Self::Drop),
-            "ig" => Ok(Self::InternetGateway),
+            "ig" => Ok(Self::InternetGateway(None)),
             lower => match lower.split_once('=') {
                 Some(("ip4", ip4s)) => {
                     let ip4 = ip4s
@@ -396,6 +399,10 @@ impl FromStr for RouterTarget {
                     cidr6s.parse().map(|x| Self::VpcSubnet(IpCidr::Ip6(x)))
                 }
 
+                Some(("ig", uuid)) => Ok(Self::InternetGateway(Some(
+                    uuid.parse::<Uuid>().map_err(|e| e.to_string())?,
+                ))),
+
                 _ => Err(format!("malformed router target: {}", lower)),
             },
         }
@@ -406,7 +413,8 @@ impl Display for RouterTarget {
     fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
         match self {
             Self::Drop => write!(f, "Drop"),
-            Self::InternetGateway => write!(f, "IG"),
+            Self::InternetGateway(None) => write!(f, "ig"),
+            Self::InternetGateway(Some(id)) => write!(f, "ig={}", id),
             Self::Ip(IpAddr::Ip4(ip4)) => write!(f, "ip4={}", ip4),
             Self::Ip(IpAddr::Ip6(ip6)) => write!(f, "ip6={}", ip6),
             Self::VpcSubnet(IpCidr::Ip4(sub4)) => write!(f, "sub4={}", sub4),
@@ -612,6 +620,7 @@ pub struct SetExternalIpsReq {
     pub port_name: String,
     pub external_ips_v4: Option<ExternalIpCfg<Ipv4Addr>>,
     pub external_ips_v6: Option<ExternalIpCfg<Ipv6Addr>>,
+    pub inet_gw_map: Option<BTreeMap<IpAddr, BTreeSet<Uuid>>>,
 }
 
 #[derive(Debug, Deserialize, Serialize)]