diff --git a/.github/buildomat/jobs/opteadm.sh b/.github/buildomat/jobs/opteadm.sh index 56133dde..3e0cd56f 100755 --- a/.github/buildomat/jobs/opteadm.sh +++ b/.github/buildomat/jobs/opteadm.sh @@ -11,6 +11,10 @@ #: "=/work/release/opteadm.release.sha256", #: ] #: +#: [[publish]] +#: series = "release" +#: name = "opteadm" +#: from_output = "/work/release/opteadm" set -o errexit set -o pipefail diff --git a/.github/buildomat/jobs/xde.sh b/.github/buildomat/jobs/xde.sh index af1aa68b..750d7535 100755 --- a/.github/buildomat/jobs/xde.sh +++ b/.github/buildomat/jobs/xde.sh @@ -21,7 +21,7 @@ #: series = "module" #: name = "xde" #: from_output = "/work/release/xde" -# +#: #: [[publish]] #: series = "module" #: name = "xde.sha256" diff --git a/Cargo.lock b/Cargo.lock index 6c0e4d9b..273d40ec 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1345,6 +1345,7 @@ dependencies = [ "smoltcp", "tabwriter", "usdt", + "uuid", "zerocopy", ] @@ -2365,6 +2366,15 @@ version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" +[[package]] +name = "uuid" +version = "1.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "81dfa00651efa65069b0b6b651f4aaa31ba9e3c3ce0137aaad053604ee7e0314" +dependencies = [ + "serde", +] + [[package]] name = "version_check" version = "0.9.5" diff --git a/Cargo.toml b/Cargo.toml index febc85c7..992a077c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -72,6 +72,7 @@ syn = "2" tabwriter = { version = "1", features = ["ansi_formatting"] } thiserror = "1.0" toml = "0.8" +uuid = { version = "1.0", default-features = false, features = ["serde"]} usdt = "0.5" version_check = "0.9" zerocopy = { version = "0.7", features = ["derive"] } diff --git a/bench/src/packet.rs b/bench/src/packet.rs index 44df84e0..97cef04e 100644 --- a/bench/src/packet.rs +++ b/bench/src/packet.rs @@ -294,7 +294,7 @@ impl BenchPacketInstance for UlpProcessInstance { router::add_entry( &g1.port, IpCidr::Ip4("0.0.0.0/0".parse().unwrap()), - RouterTarget::InternetGateway, + RouterTarget::InternetGateway(None), RouterClass::System, ) .unwrap(); @@ -303,7 +303,7 @@ impl BenchPacketInstance for UlpProcessInstance { router::add_entry( &g1.port, IpCidr::Ip6("::/0".parse().unwrap()), - RouterTarget::InternetGateway, + RouterTarget::InternetGateway(None), RouterClass::System, ) .unwrap(); diff --git a/bin/opteadm/src/bin/opteadm.rs b/bin/opteadm/src/bin/opteadm.rs index ff5f2fd2..f9d4b120 100644 --- a/bin/opteadm/src/bin/opteadm.rs +++ b/bin/opteadm/src/bin/opteadm.rs @@ -782,6 +782,10 @@ fn main() -> anyhow::Result<()> { port_name: port, external_ips_v4: Some(cfg), external_ips_v6: None, + // TODO: It'd be nice to have this user-specifiable via + // opteadm, but for now it's a purely control-plane + // concept. + inet_gw_map: None, }; hdl.set_external_ips(&req)?; } else if let Ok(cfg) = @@ -791,6 +795,8 @@ fn main() -> anyhow::Result<()> { port_name: port, external_ips_v6: Some(cfg), external_ips_v4: None, + // TODO: As above. + inet_gw_map: None, }; hdl.set_external_ips(&req)?; } else { diff --git a/lib/opte-test-utils/src/lib.rs b/lib/opte-test-utils/src/lib.rs index cabb4fc9..480b7c7e 100644 --- a/lib/opte-test-utils/src/lib.rs +++ b/lib/opte-test-utils/src/lib.rs @@ -102,6 +102,9 @@ pub const BS_IP_ADDR: Ipv6Addr = const UFT_LIMIT: Option = NonZeroU32::new(16); const TCP_LIMIT: Option = NonZeroU32::new(16); +pub const EXT_IP4: &str = "10.77.77.13"; +pub const EXT_IP6: &str = "fd00:100::1"; + pub fn ox_vpc_mac(id: [u8; 3]) -> MacAddr { MacAddr::from([0xA8, 0x40, 0x25, 0xF0 | id[0], id[1], id[2]]) } @@ -132,7 +135,7 @@ pub fn g1_cfg() -> VpcCfg { gateway_ip: "172.30.0.1".parse().unwrap(), external_ips: ExternalIpCfg { snat: Some(SNat4Cfg { - external_ip: "10.77.77.13".parse().unwrap(), + external_ip: EXT_IP4.parse().unwrap(), ports: 1025..=4096, }), ephemeral_ip: None, @@ -372,7 +375,8 @@ pub fn oxide_net_setup2( "set:firewall.rules.out=0", // * Outbound IPv4 SNAT // * Outbound IPv6 SNAT - "set:nat.rules.out=2", + // * Drop uncaught InetGw packets. + "set:nat.rules.out=3", ]; [ diff --git a/lib/oxide-vpc/Cargo.toml b/lib/oxide-vpc/Cargo.toml index 45ff291e..a4c96c72 100644 --- a/lib/oxide-vpc/Cargo.toml +++ b/lib/oxide-vpc/Cargo.toml @@ -33,12 +33,13 @@ usdt = ["opte/usdt"] illumos-sys-hdrs.workspace = true opte.workspace = true +cfg-if.workspace = true +poptrie.workspace = true serde.workspace = true smoltcp.workspace = true tabwriter = { workspace = true, optional = true } +uuid.workspace = true zerocopy.workspace = true -poptrie.workspace = true -cfg-if.workspace = true [dev-dependencies] ctor.workspace = true diff --git a/lib/oxide-vpc/src/api.rs b/lib/oxide-vpc/src/api.rs index 011908bf..9e4be6a2 100644 --- a/lib/oxide-vpc/src/api.rs +++ b/lib/oxide-vpc/src/api.rs @@ -4,6 +4,7 @@ // Copyright 2024 Oxide Computer Company +use alloc::collections::BTreeMap; use alloc::collections::BTreeSet; use alloc::string::String; use alloc::string::ToString; @@ -16,6 +17,7 @@ use illumos_sys_hdrs::datalink_id_t; pub use opte::api::*; use serde::Deserialize; use serde::Serialize; +use uuid::Uuid; /// This is the MAC address that OPTE uses to act as the virtual gateway. pub const GW_MAC_ADDR: MacAddr = @@ -347,7 +349,8 @@ impl From for GuestPhysAddr { /// * InternetGateway: Packets matching this entry are forwarded to /// the internet. In the case of the Oxide Network the IG is not an /// actual destination, but rather a configuration that determines how -/// we should NAT the flow. +/// we should NAT the flow. The address in the gateway is the source +/// address that is to be used. /// /// * Ip: Packets matching this entry are forwarded to the specified IP. /// @@ -362,7 +365,7 @@ impl From for GuestPhysAddr { #[derive(Clone, Debug, Copy, Deserialize, Serialize)] pub enum RouterTarget { Drop, - InternetGateway, + InternetGateway(Option), Ip(IpAddr), VpcSubnet(IpCidr), } @@ -374,7 +377,7 @@ impl FromStr for RouterTarget { fn from_str(s: &str) -> Result { match s.to_ascii_lowercase().as_str() { "drop" => Ok(Self::Drop), - "ig" => Ok(Self::InternetGateway), + "ig" => Ok(Self::InternetGateway(None)), lower => match lower.split_once('=') { Some(("ip4", ip4s)) => { let ip4 = ip4s @@ -396,6 +399,10 @@ impl FromStr for RouterTarget { cidr6s.parse().map(|x| Self::VpcSubnet(IpCidr::Ip6(x))) } + Some(("ig", uuid)) => Ok(Self::InternetGateway(Some( + uuid.parse::().map_err(|e| e.to_string())?, + ))), + _ => Err(format!("malformed router target: {}", lower)), }, } @@ -406,7 +413,8 @@ impl Display for RouterTarget { fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { match self { Self::Drop => write!(f, "Drop"), - Self::InternetGateway => write!(f, "IG"), + Self::InternetGateway(None) => write!(f, "ig"), + Self::InternetGateway(Some(id)) => write!(f, "ig={}", id), Self::Ip(IpAddr::Ip4(ip4)) => write!(f, "ip4={}", ip4), Self::Ip(IpAddr::Ip6(ip6)) => write!(f, "ip6={}", ip6), Self::VpcSubnet(IpCidr::Ip4(sub4)) => write!(f, "sub4={}", sub4), @@ -612,6 +620,7 @@ pub struct SetExternalIpsReq { pub port_name: String, pub external_ips_v4: Option>, pub external_ips_v6: Option>, + pub inet_gw_map: Option>>, } #[derive(Debug, Deserialize, Serialize)] diff --git a/lib/oxide-vpc/src/engine/nat.rs b/lib/oxide-vpc/src/engine/nat.rs index 3deb85ec..e7f4b900 100644 --- a/lib/oxide-vpc/src/engine/nat.rs +++ b/lib/oxide-vpc/src/engine/nat.rs @@ -4,6 +4,7 @@ // Copyright 2023 Oxide Computer Company +use super::router::RouterTargetClass; use super::router::RouterTargetInternal; use super::router::ROUTER_LAYER_NAME; use super::VpcNetwork; @@ -13,6 +14,8 @@ use crate::cfg::IpCfg; use crate::cfg::Ipv4Cfg; use crate::cfg::Ipv6Cfg; use crate::cfg::VpcCfg; +use alloc::collections::BTreeMap; +use alloc::collections::BTreeSet; use alloc::string::ToString; use alloc::sync::Arc; use alloc::vec::Vec; @@ -43,11 +46,13 @@ use opte::engine::rule::Finalized; use opte::engine::rule::Rule; use opte::engine::snat::ConcreteIpAddr; use opte::engine::snat::SNat; +use uuid::Uuid; pub const NAT_LAYER_NAME: &str = "nat"; const FLOATING_ONE_TO_ONE_NAT_PRIORITY: u16 = 5; const EPHEMERAL_ONE_TO_ONE_NAT_PRIORITY: u16 = 10; const SNAT_PRIORITY: u16 = 100; +const NO_EIP_PRIORITY: u16 = 255; // A note on concurrency correctness of accessing the `Dynamic`: // * Validation will not be triggered until table is marked dirty @@ -89,9 +94,12 @@ pub fn setup( cfg: &VpcCfg, ft_limit: NonZeroU32, ) -> Result<(), OpteError> { - // The NAT layer is rewrite layer and not a filtering one. Any - // packets that don't match should be allowed to pass through to + // The NAT layer is generally rewrite layer and not a filtering one. + // Any packets that don't match should be allowed to pass through to // the next layer. + // There is one exception: a packet with an InternetGateway target + // but no valid replacement source IP must be dropped, otherwise it will + // be forwarded to boundary services. let actions = LayerActions { actions: vec![], default_in: DefaultAction::Allow, @@ -99,7 +107,7 @@ pub fn setup( }; let mut layer = Layer::new(NAT_LAYER_NAME, pb.name(), actions, ft_limit); - let (in_rules, out_rules) = create_nat_rules(cfg)?; + let (in_rules, out_rules) = create_nat_rules(cfg, None)?; layer.set_rules(in_rules, out_rules); pb.add_layer(layer, Pos::After(ROUTER_LAYER_NAME)) } @@ -107,15 +115,39 @@ pub fn setup( #[allow(clippy::type_complexity)] fn create_nat_rules( cfg: &VpcCfg, + inet_gw_map: Option>>, ) -> Result<(Vec>, Vec>), OpteError> { let mut in_rules = vec![]; let mut out_rules = vec![]; if let Some(ipv4_cfg) = cfg.ipv4_cfg() { - setup_ipv4_nat(ipv4_cfg, &mut in_rules, &mut out_rules)?; + setup_ipv4_nat( + ipv4_cfg, + &mut in_rules, + &mut out_rules, + inet_gw_map.as_ref(), + )?; } if let Some(ipv6_cfg) = cfg.ipv6_cfg() { - setup_ipv6_nat(ipv6_cfg, &mut in_rules, &mut out_rules)?; + setup_ipv6_nat( + ipv6_cfg, + &mut in_rules, + &mut out_rules, + inet_gw_map.as_ref(), + )?; } + + // Append an additional rule to drop any InternetGateway packets + // which *did not* match an existing source IP address. This is + // expected to occur in cases where we have assigned multiple + // internet gateways but have no valid source address on a selected + // IGW. + let mut out_igw_nat_miss = Rule::new(NO_EIP_PRIORITY, Action::Deny); + out_igw_nat_miss.add_predicate(Predicate::Meta( + RouterTargetClass::KEY.to_string(), + RouterTargetClass::InternetGateway.as_meta(), + )); + out_rules.push(out_igw_nat_miss.finalize()); + Ok((in_rules, out_rules)) } @@ -124,6 +156,7 @@ fn setup_ipv4_nat( ip_cfg: &Ipv4Cfg, in_rules: &mut Vec>, out_rules: &mut Vec>, + inet_gw_map: Option<&BTreeMap>>, ) -> Result<(), OpteError> { // When it comes to NAT we always prefer using 1:1 NAT of external // IP to SNAT, preferring floating IPs over ephemeral. @@ -133,23 +166,48 @@ fn setup_ipv4_nat( let in_nat = Arc::new(InboundNat::new(ip_cfg.private_ip, verifier.clone())); let external_cfg = ip_cfg.external_ips.load(); + // Outbound IP selection needs to be gated upon which internet gateway was + // chosen during routing. + // We need to partition FIPs into separate lists based on which internet gateway + // each belongs to. This may in future extend to further SNATs from each + // attached Internet Gateway, but not today. if !external_cfg.floating_ips.is_empty() { - let mut out_nat = Rule::new( - FLOATING_ONE_TO_ONE_NAT_PRIORITY, - Action::Stateful(Arc::new(OutboundNat::new( - ip_cfg.private_ip, - &external_cfg.floating_ips, - verifier.clone(), - ))), - ); - out_nat.add_predicate(Predicate::InnerEtherType(vec![ - EtherTypeMatch::Exact(ETHER_TYPE_IPV4), - ])); - out_nat.add_predicate(Predicate::Meta( - RouterTargetInternal::KEY.to_string(), - RouterTargetInternal::InternetGateway.as_meta(), - )); - out_rules.push(out_nat.finalize()); + let mut fips_by_gw: BTreeMap, Vec> = + BTreeMap::new(); + for ip in &external_cfg.floating_ips { + let gw_mappings = + inet_gw_map.and_then(|map| map.get(&(*ip).into())).cloned(); + if let Some(igw_list) = gw_mappings { + for igw in igw_list { + let entry = fips_by_gw.entry(Some(igw)); + let ips = entry.or_default(); + ips.push(*ip); + } + } else { + let entry = fips_by_gw.entry(None); + let ips = entry.or_default(); + ips.push(*ip); + }; + } + + for (gw, fips) in fips_by_gw { + let mut out_nat = Rule::new( + FLOATING_ONE_TO_ONE_NAT_PRIORITY, + Action::Stateful(Arc::new(OutboundNat::new( + ip_cfg.private_ip, + &fips[..], + verifier.clone(), + ))), + ); + out_nat.add_predicate(Predicate::InnerEtherType(vec![ + EtherTypeMatch::Exact(ETHER_TYPE_IPV4), + ])); + out_nat.add_predicate(Predicate::Meta( + RouterTargetInternal::KEY.to_string(), + RouterTargetInternal::InternetGateway(gw).as_meta(), + )); + out_rules.push(out_nat.finalize()); + } // 1:1 NAT inbound packets destined for external IP. let mut in_nat = Rule::new( @@ -168,22 +226,34 @@ fn setup_ipv4_nat( if let Some(ip4) = external_cfg.ephemeral_ip { // 1:1 NAT outbound packets destined for internet gateway. - let mut out_nat = Rule::new( - EPHEMERAL_ONE_TO_ONE_NAT_PRIORITY, - Action::Stateful(Arc::new(OutboundNat::new( - ip_cfg.private_ip, - &[ip4], - verifier, - ))), - ); - out_nat.add_predicate(Predicate::InnerEtherType(vec![ - EtherTypeMatch::Exact(ETHER_TYPE_IPV4), - ])); - out_nat.add_predicate(Predicate::Meta( - RouterTargetInternal::KEY.to_string(), - RouterTargetInternal::InternetGateway.as_meta(), - )); - out_rules.push(out_nat.finalize()); + let igw_matches = match inet_gw_map { + Some(inet_gw_map) => match inet_gw_map.get(&(ip4.into())) { + Some(igw_set) => { + igw_set.iter().copied().map(Option::Some).collect() + } + None => vec![None], + }, + None => vec![None], + }; + + for igw_id in igw_matches { + let mut out_nat = Rule::new( + EPHEMERAL_ONE_TO_ONE_NAT_PRIORITY, + Action::Stateful(Arc::new(OutboundNat::new( + ip_cfg.private_ip, + &[ip4], + verifier.clone(), + ))), + ); + out_nat.add_predicate(Predicate::InnerEtherType(vec![ + EtherTypeMatch::Exact(ETHER_TYPE_IPV4), + ])); + out_nat.add_predicate(Predicate::Meta( + RouterTargetInternal::KEY.to_string(), + RouterTargetInternal::InternetGateway(igw_id).as_meta(), + )); + out_rules.push(out_nat.finalize()); + } // 1:1 NAT inbound packets destined for external IP. let mut in_nat = Rule::new( @@ -197,23 +267,39 @@ fn setup_ipv4_nat( } if let Some(snat_cfg) = &external_cfg.snat { + let igw_matches = match inet_gw_map { + Some(inet_gw_map) => { + match inet_gw_map.get(&(snat_cfg.external_ip.into())) { + Some(igw_set) => { + igw_set.iter().copied().map(Option::Some).collect() + } + None => vec![None], + } + } + None => vec![None], + }; + let snat = SNat::new(ip_cfg.private_ip); snat.add( ip_cfg.private_ip, snat_cfg.external_ip, snat_cfg.ports.clone(), ); - let mut rule = - Rule::new(SNAT_PRIORITY, Action::Stateful(Arc::new(snat))); + let snat = Arc::new(snat); - rule.add_predicate(Predicate::InnerEtherType(vec![ - EtherTypeMatch::Exact(ETHER_TYPE_IPV4), - ])); - rule.add_predicate(Predicate::Meta( - RouterTargetInternal::KEY.to_string(), - RouterTargetInternal::InternetGateway.as_meta(), - )); - out_rules.push(rule.finalize()); + for igw_id in igw_matches { + let mut rule = + Rule::new(SNAT_PRIORITY, Action::Stateful(snat.clone())); + + rule.add_predicate(Predicate::InnerEtherType(vec![ + EtherTypeMatch::Exact(ETHER_TYPE_IPV4), + ])); + rule.add_predicate(Predicate::Meta( + RouterTargetInternal::KEY.to_string(), + RouterTargetInternal::InternetGateway(igw_id).as_meta(), + )); + out_rules.push(rule.finalize()); + } } Ok(()) } @@ -222,6 +308,7 @@ fn setup_ipv6_nat( ip_cfg: &Ipv6Cfg, in_rules: &mut Vec>, out_rules: &mut Vec>, + inet_gw_map: Option<&BTreeMap>>, ) -> Result<(), OpteError> { // When it comes to NAT we always prefer using 1:1 NAT of external // IP to SNAT, preferring floating IPs over ephemeral. @@ -231,23 +318,45 @@ fn setup_ipv6_nat( let in_nat = Arc::new(InboundNat::new(ip_cfg.private_ip, verifier.clone())); let external_cfg = ip_cfg.external_ips.load(); + // See `setup_ipv4_nat` for an explanation on partitioning FIPs + // by internet gateway ID. if !external_cfg.floating_ips.is_empty() { - let mut out_nat = Rule::new( - FLOATING_ONE_TO_ONE_NAT_PRIORITY, - Action::Stateful(Arc::new(OutboundNat::new( - ip_cfg.private_ip, - &external_cfg.floating_ips, - verifier.clone(), - ))), - ); - out_nat.add_predicate(Predicate::InnerEtherType(vec![ - EtherTypeMatch::Exact(ETHER_TYPE_IPV6), - ])); - out_nat.add_predicate(Predicate::Meta( - RouterTargetInternal::KEY.to_string(), - RouterTargetInternal::InternetGateway.as_meta(), - )); - out_rules.push(out_nat.finalize()); + let mut fips_by_gw: BTreeMap, Vec> = + BTreeMap::new(); + for ip in &external_cfg.floating_ips { + let gw_mappings = + inet_gw_map.and_then(|map| map.get(&(*ip).into())).cloned(); + if let Some(igw_list) = gw_mappings { + for igw in igw_list { + let entry = fips_by_gw.entry(Some(igw)); + let ips = entry.or_default(); + ips.push(*ip); + } + } else { + let entry = fips_by_gw.entry(None); + let ips = entry.or_default(); + ips.push(*ip); + }; + } + + for (gw, fips) in fips_by_gw { + let mut out_nat = Rule::new( + FLOATING_ONE_TO_ONE_NAT_PRIORITY, + Action::Stateful(Arc::new(OutboundNat::new( + ip_cfg.private_ip, + &fips[..], + verifier.clone(), + ))), + ); + out_nat.add_predicate(Predicate::InnerEtherType(vec![ + EtherTypeMatch::Exact(ETHER_TYPE_IPV6), + ])); + out_nat.add_predicate(Predicate::Meta( + RouterTargetInternal::KEY.to_string(), + RouterTargetInternal::InternetGateway(gw).as_meta(), + )); + out_rules.push(out_nat.finalize()); + } // 1:1 NAT inbound packets destined for external IP. let mut in_nat = Rule::new( @@ -266,22 +375,34 @@ fn setup_ipv6_nat( if let Some(ip6) = external_cfg.ephemeral_ip { // 1:1 NAT outbound packets destined for internet gateway. - let mut out_nat = Rule::new( - EPHEMERAL_ONE_TO_ONE_NAT_PRIORITY, - Action::Stateful(Arc::new(OutboundNat::new( - ip_cfg.private_ip, - &[ip6], - verifier, - ))), - ); - out_nat.add_predicate(Predicate::InnerEtherType(vec![ - EtherTypeMatch::Exact(ETHER_TYPE_IPV6), - ])); - out_nat.add_predicate(Predicate::Meta( - RouterTargetInternal::KEY.to_string(), - RouterTargetInternal::InternetGateway.as_meta(), - )); - out_rules.push(out_nat.finalize()); + let igw_matches = match inet_gw_map { + Some(inet_gw_map) => match inet_gw_map.get(&(ip6.into())) { + Some(igw_set) => { + igw_set.iter().copied().map(Option::Some).collect() + } + None => vec![None], + }, + None => vec![None], + }; + + for igw_id in igw_matches { + let mut out_nat = Rule::new( + EPHEMERAL_ONE_TO_ONE_NAT_PRIORITY, + Action::Stateful(Arc::new(OutboundNat::new( + ip_cfg.private_ip, + &[ip6], + verifier.clone(), + ))), + ); + out_nat.add_predicate(Predicate::InnerEtherType(vec![ + EtherTypeMatch::Exact(ETHER_TYPE_IPV6), + ])); + out_nat.add_predicate(Predicate::Meta( + RouterTargetInternal::KEY.to_string(), + RouterTargetInternal::InternetGateway(igw_id).as_meta(), + )); + out_rules.push(out_nat.finalize()); + } // 1:1 NAT inbound packets destined for external IP. let mut in_nat = Rule::new( @@ -295,23 +416,39 @@ fn setup_ipv6_nat( } if let Some(ref snat_cfg) = external_cfg.snat { + let igw_matches = match inet_gw_map { + Some(inet_gw_map) => { + match inet_gw_map.get(&(snat_cfg.external_ip.into())) { + Some(igw_set) => { + igw_set.iter().copied().map(Option::Some).collect() + } + None => vec![None], + } + } + None => vec![None], + }; + let snat = SNat::new(ip_cfg.private_ip); snat.add( ip_cfg.private_ip, snat_cfg.external_ip, snat_cfg.ports.clone(), ); - let mut rule = - Rule::new(SNAT_PRIORITY, Action::Stateful(Arc::new(snat))); + let snat = Arc::new(snat); - rule.add_predicate(Predicate::InnerEtherType(vec![ - EtherTypeMatch::Exact(ETHER_TYPE_IPV6), - ])); - rule.add_predicate(Predicate::Meta( - RouterTargetInternal::KEY.to_string(), - RouterTargetInternal::InternetGateway.as_meta(), - )); - out_rules.push(rule.finalize()); + for igw_id in igw_matches { + let mut rule = + Rule::new(SNAT_PRIORITY, Action::Stateful(snat.clone())); + + rule.add_predicate(Predicate::InnerEtherType(vec![ + EtherTypeMatch::Exact(ETHER_TYPE_IPV6), + ])); + rule.add_predicate(Predicate::Meta( + RouterTargetInternal::KEY.to_string(), + RouterTargetInternal::InternetGateway(igw_id).as_meta(), + )); + out_rules.push(rule.finalize()); + } } Ok(()) } @@ -348,6 +485,6 @@ pub fn set_nat_rules( _ => return Err(OpteError::InvalidIpCfg), } - let (in_rules, out_rules) = create_nat_rules(cfg)?; + let (in_rules, out_rules) = create_nat_rules(cfg, req.inet_gw_map)?; port.set_rules_soft(NAT_LAYER_NAME, in_rules, out_rules) } diff --git a/lib/oxide-vpc/src/engine/overlay.rs b/lib/oxide-vpc/src/engine/overlay.rs index 730ac41e..a3c70948 100644 --- a/lib/oxide-vpc/src/engine/overlay.rs +++ b/lib/oxide-vpc/src/engine/overlay.rs @@ -211,7 +211,7 @@ impl StaticAction for EncapAction { // The router layer determines a RouterTarget and stores it in // the meta map. We need to map this virtual target to a // physical one. - let target_str = match action_meta.get(RouterTargetInternal::KEY) { + let target_str = match action_meta.get(RouterTargetInternal::IP_KEY) { Some(val) => val, None => { // This should never happen. The router should always @@ -237,7 +237,7 @@ impl StaticAction for EncapAction { }; let phys_target = match target { - RouterTargetInternal::InternetGateway => { + RouterTargetInternal::InternetGateway(_) => { match self.v2b.get(&flow_id.dst_ip()) { Some(phys) => { // Hash the packet onto a route target. This is a very @@ -318,7 +318,7 @@ impl StaticAction for EncapAction { } }; - Ok(AllowOrDeny::Allow(HdrTransform { + let tfrm = HdrTransform { name: ENCAP_NAME.to_string(), // We leave the outer src/dst up to the driver. outer_ether: HeaderAction::Push( @@ -358,7 +358,9 @@ impl StaticAction for EncapAction { PhantomData, ), ..Default::default() - })) + }; + + Ok(AllowOrDeny::Allow(tfrm)) } fn implicit_preds(&self) -> (Vec, Vec) { diff --git a/lib/oxide-vpc/src/engine/router.rs b/lib/oxide-vpc/src/engine/router.rs index a3b8b78b..9b855344 100644 --- a/lib/oxide-vpc/src/engine/router.rs +++ b/lib/oxide-vpc/src/engine/router.rs @@ -47,6 +47,7 @@ use opte::engine::rule::Finalized; use opte::engine::rule::MetaAction; use opte::engine::rule::ModMetaResult; use opte::engine::rule::Rule; +use uuid::Uuid; pub const ROUTER_LAYER_NAME: &str = "router"; @@ -57,17 +58,43 @@ pub const ROUTER_LAYER_NAME: &str = "router"; // remaining possible targets. #[derive(Clone, Copy, Debug, Eq, PartialEq)] pub enum RouterTargetInternal { - InternetGateway, + // The selected internet gateway determines a packet's chosen source + // address during NAT. We don't necessarily *know* the ID of this + // gateway. + InternetGateway(Option), Ip(IpAddr), VpcSubnet(IpCidr), } +impl RouterTargetInternal { + pub const IP_KEY: &'static str = "router-target-ip"; + pub const GENERIC_META: &'static str = "ig"; + + pub fn generic_meta(&self) -> String { + Self::GENERIC_META.to_string() + } + + pub fn ip_key(&self) -> String { + Self::IP_KEY.to_string() + } + + pub fn class(&self) -> RouterTargetClass { + match self { + RouterTargetInternal::InternetGateway(_) => { + RouterTargetClass::InternetGateway + } + RouterTargetInternal::Ip(_) => RouterTargetClass::Ip, + RouterTargetInternal::VpcSubnet(_) => RouterTargetClass::VpcSubnet, + } + } +} + impl ActionMetaValue for RouterTargetInternal { const KEY: &'static str = "router-target"; fn from_meta(s: &str) -> Result { match s { - "ig" => Ok(Self::InternetGateway), + "ig" => Ok(Self::InternetGateway(None)), _ => match s.split_once('=') { Some(("ip4", ip4_s)) => { let ip4 = ip4_s.parse::()?; @@ -89,6 +116,11 @@ impl ActionMetaValue for RouterTargetInternal { Ok(Self::VpcSubnet(IpCidr::Ip6(cidr6))) } + Some(("ig", ig)) => { + let ig = ig.parse::().map_err(|e| e.to_string())?; + Ok(Self::InternetGateway(Some(ig))) + } + _ => Err(format!("bad router target: {}", s)), }, } @@ -96,7 +128,10 @@ impl ActionMetaValue for RouterTargetInternal { fn as_meta(&self) -> String { match self { - Self::InternetGateway => "ig".to_string(), + Self::InternetGateway(ip) => match ip { + Some(ip) => format!("ig={}", ip), + None => String::from("ig"), + }, Self::Ip(IpAddr::Ip4(ip4)) => format!("ip4={}", ip4), Self::Ip(IpAddr::Ip6(ip6)) => format!("ip6={}", ip6), Self::VpcSubnet(IpCidr::Ip4(cidr4)) => format!("sub4={}", cidr4), @@ -108,7 +143,7 @@ impl ActionMetaValue for RouterTargetInternal { impl fmt::Display for RouterTargetInternal { fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { let s = match self { - Self::InternetGateway => "IG".to_string(), + Self::InternetGateway(addr) => format!("IG({:?})", addr), Self::Ip(addr) => format!("IP: {}", addr), Self::VpcSubnet(sub) => format!("Subnet: {}", sub), }; @@ -116,6 +151,44 @@ impl fmt::Display for RouterTargetInternal { } } +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum RouterTargetClass { + InternetGateway, + Ip, + VpcSubnet, +} + +impl ActionMetaValue for RouterTargetClass { + const KEY: &'static str = "router-target-class"; + + fn from_meta(s: &str) -> Result { + match s { + "ig" => Ok(Self::InternetGateway), + "ip" => Ok(Self::Ip), + "subnet" => Ok(Self::VpcSubnet), + _ => Err(format!("bad router target class: {}", s)), + } + } + + fn as_meta(&self) -> String { + match self { + Self::InternetGateway => "ig".into(), + Self::Ip => "ip".into(), + Self::VpcSubnet => "subnet".into(), + } + } +} + +impl fmt::Display for RouterTargetClass { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + match self { + Self::InternetGateway => write!(f, "IG"), + Self::Ip => write!(f, "IP"), + Self::VpcSubnet => write!(f, "Subnet"), + } + } +} + // Return the priority for a given IP subnet. The priority is based on // the subnet's prefix length and the type of router the rule belongs to. // Specifically, it is given the following value: @@ -198,6 +271,8 @@ fn valid_router_dest_target_pair(dest: &IpCidr, target: &RouterTarget) -> bool { (&dest, &target), // Anything can be dropped (_, RouterTarget::Drop) | + // Internet gateways are valid for any IP family. + (_, RouterTarget::InternetGateway(_)) | // IPv4 destination, IPv4 address (IpCidr::Ip4(_), RouterTarget::Ip(IpAddr::Ip4(_))) | // IPv4 destination, IPv4 subnet @@ -205,11 +280,7 @@ fn valid_router_dest_target_pair(dest: &IpCidr, target: &RouterTarget) -> bool { // IPv6 destination, IPv6 address (IpCidr::Ip6(_), RouterTarget::Ip(IpAddr::Ip6(_))) | // IPv6 destination, IPv6 subnet - (IpCidr::Ip6(_), RouterTarget::VpcSubnet(IpCidr::Ip6(_))) | - // IPv4 destination, IPv4 Internet Gateway - (IpCidr::Ip4(_), RouterTarget::InternetGateway) | - // IPv6 destination, IPv6 Internet Gateway - (IpCidr::Ip6(_), RouterTarget::InternetGateway) + (IpCidr::Ip6(_), RouterTarget::VpcSubnet(IpCidr::Ip6(_))) ) } @@ -239,7 +310,7 @@ fn make_rule( (predicate, Action::Deny) } - RouterTarget::InternetGateway => { + RouterTarget::InternetGateway(id) => { let predicate = match dest { IpCidr::Ip4(ip4) => { Predicate::InnerDstIp4(vec![Ipv4AddrMatch::Prefix(ip4)]) @@ -250,7 +321,7 @@ fn make_rule( } }; let action = Action::Meta(Arc::new(RouterAction::new( - RouterTargetInternal::InternetGateway, + RouterTargetInternal::InternetGateway(id), ))); (predicate, action) } @@ -291,6 +362,7 @@ fn make_rule( let priority = compute_rule_priority(&dest, class); let mut rule = Rule::new(priority, action); rule.add_predicate(predicate); + Ok(rule.finalize()) } @@ -382,11 +454,13 @@ impl MetaAction for RouterAction { _flow_id: &InnerFlowId, meta: &mut ActionMeta, ) -> ModMetaResult { - // No target entry should currently exist in the metadata; it - // would be a bug. However, because of the dynamic nature of - // metadata we don't have an easy way to enforce this - // constraint in the type system. - meta.insert(self.target.key(), self.target.as_meta()); + // TODO: I don't think we need IP_KEY. + if let RouterTargetInternal::InternetGateway(_) = self.target { + meta.insert(self.target.key(), self.target.as_meta()); + } + meta.insert(self.target.ip_key(), self.target.as_meta()); + let rt_class = self.target.class(); + meta.insert(rt_class.key(), rt_class.as_meta()); Ok(AllowOrDeny::Allow(())) } } diff --git a/lib/oxide-vpc/tests/firewall_tests.rs b/lib/oxide-vpc/tests/firewall_tests.rs index 50effe53..345b8f61 100644 --- a/lib/oxide-vpc/tests/firewall_tests.rs +++ b/lib/oxide-vpc/tests/firewall_tests.rs @@ -156,7 +156,7 @@ fn firewall_vni_inbound() { // could prove useful in other tests. let g1_ext_ip = "10.77.78.9".parse().unwrap(); g1_cfg.set_ext_ipv4(g1_ext_ip); - let custom = ["set:nat.rules.in=1", "set:nat.rules.out=3"]; + let custom = ["set:nat.rules.in=1", "set:nat.rules.out=4"]; let mut g1 = oxide_net_setup2("g1_port", &g1_cfg, None, None, Some(&custom)); g1.port.start(); @@ -332,7 +332,7 @@ fn firewall_external_inbound() { let mut g1_cfg = g1_cfg(); let g1_ext_ip = "10.77.78.9".parse().unwrap(); g1_cfg.set_ext_ipv4(g1_ext_ip); - let custom = ["set:nat.rules.in=1", "set:nat.rules.out=3"]; + let custom = ["set:nat.rules.in=1", "set:nat.rules.out=4"]; let mut g1 = oxide_net_setup2("g1_port", &g1_cfg, None, None, Some(&custom)); g1.port.start(); diff --git a/lib/oxide-vpc/tests/integration_tests.rs b/lib/oxide-vpc/tests/integration_tests.rs index e22f3d94..74d70e31 100644 --- a/lib/oxide-vpc/tests/integration_tests.rs +++ b/lib/oxide-vpc/tests/integration_tests.rs @@ -67,8 +67,10 @@ use smoltcp::wire::NdiscNeighborFlags; use smoltcp::wire::NdiscRepr; use smoltcp::wire::NdiscRouterFlags; use smoltcp::wire::RawHardwareAddress; +use std::collections::BTreeMap; use std::prelude::v1::*; use std::time::Duration; +use uuid::Uuid; use zerocopy::AsBytes; const IP4_SZ: usize = EtherHdr::SIZE + Ipv4Hdr::BASE_SIZE; @@ -709,7 +711,7 @@ fn guest_to_internet_ipv4() { router::add_entry( &g1.port, IpCidr::Ip4("0.0.0.0/0".parse().unwrap()), - RouterTarget::InternetGateway, + RouterTarget::InternetGateway(None), RouterClass::System, ) .unwrap(); @@ -833,7 +835,7 @@ fn guest_to_internet_ipv6() { router::add_entry( &g1.port, IpCidr::Ip6("::/0".parse().unwrap()), - RouterTarget::InternetGateway, + RouterTarget::InternetGateway(None), RouterClass::System, ) .unwrap(); @@ -1031,7 +1033,7 @@ fn multi_external_ip_setup( router::add_entry( &g1.port, IpCidr::Ip6("::/0".parse().unwrap()), - RouterTarget::InternetGateway, + RouterTarget::InternetGateway(None), RouterClass::System, ) .unwrap(); @@ -1039,7 +1041,7 @@ fn multi_external_ip_setup( router::add_entry( &g1.port, IpCidr::Ip4("0.0.0.0/0".parse().unwrap()), - RouterTarget::InternetGateway, + RouterTarget::InternetGateway(None), RouterClass::System, ) .unwrap(); @@ -1354,6 +1356,10 @@ fn external_ip_epoch_affinity_preserved() { port_name: g1.port.name().to_string(), external_ips_v4: None, external_ips_v6: None, + + // This test does not focus on controlling EIP selection + // based on destination prefix. + inet_gw_map: None, }; for ext_ip in [ext_v4[0].into(), ext_v6[0].into()] { @@ -1401,7 +1407,7 @@ fn external_ip_epoch_affinity_preserved() { // since that won't affect the internal flowtable for NAT. // ==================================================================== nat::set_nat_rules(&g1.cfg, &g1.port, req.clone()).unwrap(); - update!(g1, ["incr:epoch", "set:nat.rules.in=4, nat.rules.out=6",]); + update!(g1, ["incr:epoch", "set:nat.rules.in=4, nat.rules.out=7",]); // ================================================================ // The reply packet must still originate from the ephemeral port @@ -1465,13 +1471,17 @@ fn external_ip_reconfigurable() { port_name: g1.port.name().to_string(), external_ips_v4: new_v4_cfg, external_ips_v6: new_v6_cfg, + + // This test does not focus on controlling EIP selection + // based on destination prefix. + inet_gw_map: None, }; nat::set_nat_rules(&g1.cfg, &g1.port, req).unwrap(); update!( g1, [ "incr:epoch", - "set:nat.rules.in=2, nat.rules.out=4", + "set:nat.rules.in=2, nat.rules.out=5", "set:firewall.flows.in=2, firewall.flows.out=2", ] ); @@ -1650,7 +1660,7 @@ fn snat_icmp_shared_echo_rewrite(dst_ip: IpAddr) { router::add_entry( &g1.port, IpCidr::Ip6("::/0".parse().unwrap()), - RouterTarget::InternetGateway, + RouterTarget::InternetGateway(None), RouterClass::System, ) .unwrap(); @@ -1658,7 +1668,7 @@ fn snat_icmp_shared_echo_rewrite(dst_ip: IpAddr) { router::add_entry( &g1.port, IpCidr::Ip4("0.0.0.0/0".parse().unwrap()), - RouterTarget::InternetGateway, + RouterTarget::InternetGateway(None), RouterClass::System, ) .unwrap(); @@ -2541,7 +2551,7 @@ fn outbound_ndp_dropped() { router::add_entry( &g1.port, IpCidr::Ip6("::/0".parse().unwrap()), - RouterTarget::InternetGateway, + RouterTarget::InternetGateway(None), RouterClass::System, ) .unwrap(); @@ -3044,7 +3054,7 @@ fn uft_lft_invalidation_out() { router::add_entry( &g1.port, IpCidr::Ip4("0.0.0.0/0".parse().unwrap()), - RouterTarget::InternetGateway, + RouterTarget::InternetGateway(None), RouterClass::System, ) .unwrap(); @@ -3131,7 +3141,7 @@ fn uft_lft_invalidation_in() { router::add_entry( &g1.port, IpCidr::Ip4("0.0.0.0/0".parse().unwrap()), - RouterTarget::InternetGateway, + RouterTarget::InternetGateway(None), RouterClass::System, ) .unwrap(); @@ -3453,7 +3463,7 @@ fn tcp_outbound() { router::add_entry( &g1.port, IpCidr::Ip4("0.0.0.0/0".parse().unwrap()), - RouterTarget::InternetGateway, + RouterTarget::InternetGateway(None), RouterClass::System, ) .unwrap(); @@ -3515,7 +3525,7 @@ fn early_tcp_invalidation() { router::add_entry( &g1.port, IpCidr::Ip4("0.0.0.0/0".parse().unwrap()), - RouterTarget::InternetGateway, + RouterTarget::InternetGateway(None), RouterClass::System, ) .unwrap(); @@ -3651,6 +3661,88 @@ fn early_tcp_invalidation() { assert_eq!(TcpState::SynSent, g1.port.tcp_state(&flow).unwrap()); } +#[test] +fn ephemeral_ip_preferred_over_snat_outbound() { + let ip_cfg = IpCfg::DualStack { + ipv4: Ipv4Cfg { + vpc_subnet: "172.30.0.0/22".parse().unwrap(), + private_ip: "172.30.0.5".parse().unwrap(), + gateway_ip: "172.30.0.1".parse().unwrap(), + external_ips: ExternalIpCfg { + snat: Some(SNat4Cfg { + external_ip: "10.77.77.13".parse().unwrap(), + ports: 1025..=4096, + }), + ephemeral_ip: Some("10.60.1.20".parse().unwrap()), + floating_ips: vec![], + }, + }, + ipv6: Ipv6Cfg { + vpc_subnet: "fd00::/64".parse().unwrap(), + private_ip: "fd00::5".parse().unwrap(), + gateway_ip: "fd00::1".parse().unwrap(), + external_ips: ExternalIpCfg { + snat: Some(SNat6Cfg { + external_ip: "2001:db8::1".parse().unwrap(), + ports: 1025..=4096, + }), + ephemeral_ip: None, + floating_ips: vec![], + }, + }, + }; + + let g1_cfg = g1_cfg2(ip_cfg); + let mut g1 = oxide_net_setup("g1_port", &g1_cfg, None, None); + g1.port.start(); + set!(g1, "port_state=running"); + + // Add default route. + router::add_entry( + &g1.port, + IpCidr::Ip4("0.0.0.0/0".parse().unwrap()), + RouterTarget::InternetGateway(None), + RouterClass::System, + ) + .unwrap(); + incr!(g1, ["epoch", "router.rules.out"]); + + let client_ip = "52.10.128.69".parse().unwrap(); + + let data = b"reunion"; + let mut pkt1 = gen_icmpv4_echo_req( + g1_cfg.guest_mac, + g1_cfg.gateway_mac, + g1_cfg.ipv4().private_ip, + client_ip, + 7777, + 1, + data, + 1, + ); + + // Process the packet through our port. It should be allowed through: + // we have a V2P mapping for the target guest, and a route for the other + // subnet. + let res = g1.port.process(Out, &mut pkt1, ActionMeta::new()); + assert!(matches!(res, Ok(ProcessResult::Modified))); + + incr!( + g1, + [ + "firewall.flows.in, firewall.flows.out", + "stats.port.out_modified, stats.port.out_uft_miss, uft.out", + "nat.flows.in, nat.flows.out", + ] + ); + + assert_eq!( + pkt1.meta().inner_ip4().unwrap().src, + "10.60.1.20".parse().unwrap(), + "did not choose assigned ephemeral IP" + ); +} + // Verify TCP state transitions in relation to an inbound connection // (the "passive open"). In this case the client is external, and the // guest is the server. @@ -3698,7 +3790,7 @@ fn tcp_inbound() { router::add_entry( &g1.port, IpCidr::Ip4("0.0.0.0/0".parse().unwrap()), - RouterTarget::InternetGateway, + RouterTarget::InternetGateway(None), RouterClass::System, ) .unwrap(); @@ -3964,7 +4056,7 @@ fn no_panic_on_flow_table_full() { router::add_entry( &g1.port, IpCidr::Ip4("0.0.0.0/0".parse().unwrap()), - RouterTarget::InternetGateway, + RouterTarget::InternetGateway(None), RouterClass::System, ) .unwrap(); @@ -4013,8 +4105,8 @@ fn intra_subnet_routes_with_custom() { let cidr = IpCidr::Ip4("172.30.4.0/22".parse().unwrap()); router::add_entry( &g1.port, - cidr.clone(), - RouterTarget::VpcSubnet(cidr.clone()), + cidr, + RouterTarget::VpcSubnet(cidr), RouterClass::System, ) .unwrap(); @@ -4063,13 +4155,8 @@ fn intra_subnet_routes_with_custom() { // Suppose the user now installs a 'custom' route in the first subnet to // drop traffic towards the second subnet. This rule must take priority. - router::add_entry( - &g1.port, - cidr.clone(), - RouterTarget::Drop, - RouterClass::Custom, - ) - .unwrap(); + router::add_entry(&g1.port, cidr, RouterTarget::Drop, RouterClass::Custom) + .unwrap(); incr!(g1, ["epoch", "router.rules.out"]); let mut pkt2 = gen_icmpv4_echo_req( g1_cfg.guest_mac, @@ -4098,13 +4185,8 @@ fn intra_subnet_routes_with_custom() { ); // When the user removes this rule, traffic may flow again to subnet 2. - router::del_entry( - &g1.port, - cidr.clone(), - RouterTarget::Drop, - RouterClass::Custom, - ) - .unwrap(); + router::del_entry(&g1.port, cidr, RouterTarget::Drop, RouterClass::Custom) + .unwrap(); update!(g1, ["incr:epoch", "decr:router.rules.out"]); let mut pkt3 = gen_icmpv4_echo_req( g1_cfg.guest_mac, @@ -4144,7 +4226,7 @@ fn port_as_router_target() { let dst_ip: Ipv4Addr = "192.168.0.1".parse().unwrap(); router::add_entry( &g1.port, - cidr.clone(), + cidr, RouterTarget::Ip(g2_cfg.ipv4().private_ip.into()), RouterClass::Custom, ) @@ -4225,3 +4307,304 @@ fn port_as_router_target() { let res = g1.port.process(In, &mut pkt2, ActionMeta::new()); assert!(matches!(res, Ok(ProcessResult::Modified))); } + +#[test] +fn select_eip_conditioned_on_igw() { + // RFD 21 Internet Gateways are used as a mechanism to narrow + // down the set of valid source IPs that an outbound packet may + // choose from, conditioned on a packet's destination network. + // + // To do this, the control plane is responsible for installing + // IGW rules with UUID associations, and then determining which + // external IPs are associated with each IGW. + let default_igw = Uuid::from_u128(1); + let custom_igw0 = Uuid::from_u128(2); + let custom_igw1 = Uuid::from_u128(3); + let ipless_igw = Uuid::from_u128(4); + + // The control plane may have several IGWs associated with a given + // pool. Accordingly, there is a chance that an IP might be a valid choice + // on several prefixes. + let all_ips_igw = Uuid::from_u128(5); + + // To test this, we want to set up a port such that: + // * It has an ephemeral IP in IGW 1. + // - If we target 0.0.0.0/0, we choose the eph IP 192.168.0.1 . + // * It has FIPs across IGWs 2 [dst 1.1.1.0/24], 3 [dst 2.2.2.0/24]. + // - IGW 2 has FIPs 192.168.0.2, 192.168.0.3. Either will be picked. + // - IGW 3 has FIP 192.168.0.4. + // * It has no EIP in IGW3 [dst 3.3.3.0/24]. + // - Packets sent here are denied -- we have no valid NAT IPs for this + // outbound traffic. + // * All EIPs are valid on IGW4. + // - Packets will choose a random FIP, by priority ordering. + + let ip_cfg = IpCfg::DualStack { + ipv4: Ipv4Cfg { + vpc_subnet: "172.30.0.0/22".parse().unwrap(), + private_ip: "172.30.0.5".parse().unwrap(), + gateway_ip: "172.30.0.1".parse().unwrap(), + external_ips: ExternalIpCfg { + snat: Some(SNat4Cfg { + external_ip: "10.77.77.13".parse().unwrap(), + ports: 1025..=4096, + }), + ephemeral_ip: Some("192.168.0.1".parse().unwrap()), + floating_ips: vec![ + "192.168.0.2".parse().unwrap(), + "192.168.0.3".parse().unwrap(), + "192.168.0.4".parse().unwrap(), + ], + }, + }, + // Not really testing V6 here. Same principles apply. + ipv6: Ipv6Cfg { + vpc_subnet: "fd00::/64".parse().unwrap(), + private_ip: "fd00::5".parse().unwrap(), + gateway_ip: "fd00::1".parse().unwrap(), + external_ips: ExternalIpCfg { + snat: Some(SNat6Cfg { + external_ip: "2001:db8::1".parse().unwrap(), + ports: 1025..=4096, + }), + ephemeral_ip: None, + floating_ips: vec![], + }, + }, + }; + + // let ip_cfg = IpCfg::Ipv4( + // Ipv4Cfg { + // vpc_subnet: "172.30.0.0/22".parse().unwrap(), + // private_ip: "172.30.0.5".parse().unwrap(), + // gateway_ip: "172.30.0.1".parse().unwrap(), + // external_ips: ExternalIpCfg { + // snat: Some(SNat4Cfg { + // external_ip: "10.77.77.13".parse().unwrap(), + // ports: 1025..=4096, + // }), + // ephemeral_ip: Some("192.168.0.1".parse().unwrap()), + // floating_ips: vec![ + // "192.168.0.2".parse().unwrap(), + // "192.168.0.3".parse().unwrap(), + // "192.168.0.4".parse().unwrap(), + // ], + // }, + // }); + + let g1_cfg = g1_cfg2(ip_cfg); + let mut g1 = oxide_net_setup("g1_port", &g1_cfg, None, None); + g1.port.start(); + set!(g1, "port_state=running"); + + // Add default route. + router::add_entry( + &g1.port, + IpCidr::Ip4("0.0.0.0/0".parse().unwrap()), + RouterTarget::InternetGateway(Some(default_igw)), + RouterClass::System, + ) + .unwrap(); + incr!(g1, ["epoch", "router.rules.out"]); + + // Add custom inetgw routes. + router::add_entry( + &g1.port, + IpCidr::Ip4("1.1.1.0/24".parse().unwrap()), + RouterTarget::InternetGateway(Some(custom_igw0)), + RouterClass::Custom, + ) + .unwrap(); + incr!(g1, ["epoch", "router.rules.out"]); + router::add_entry( + &g1.port, + IpCidr::Ip4("2.2.2.0/24".parse().unwrap()), + RouterTarget::InternetGateway(Some(custom_igw1)), + RouterClass::Custom, + ) + .unwrap(); + incr!(g1, ["epoch", "router.rules.out"]); + router::add_entry( + &g1.port, + IpCidr::Ip4("3.3.3.0/24".parse().unwrap()), + RouterTarget::InternetGateway(Some(ipless_igw)), + RouterClass::Custom, + ) + .unwrap(); + incr!(g1, ["epoch", "router.rules.out"]); + router::add_entry( + &g1.port, + IpCidr::Ip4("4.4.4.0/24".parse().unwrap()), + RouterTarget::InternetGateway(Some(all_ips_igw)), + RouterClass::Custom, + ) + .unwrap(); + incr!(g1, ["epoch", "router.rules.out"]); + + // ==================================================================== + // Install new config. + // ==================================================================== + let mut inet_gw_map: BTreeMap<_, _> = Default::default(); + inet_gw_map.insert( + g1_cfg.ipv4_cfg().unwrap().external_ips.ephemeral_ip.unwrap().into(), + [default_igw, all_ips_igw].into_iter().collect(), + ); + inet_gw_map.insert( + g1_cfg.ipv4_cfg().unwrap().external_ips.floating_ips[0].into(), + [custom_igw0, all_ips_igw].into_iter().collect(), + ); + inet_gw_map.insert( + g1_cfg.ipv4_cfg().unwrap().external_ips.floating_ips[1].into(), + [custom_igw0, all_ips_igw].into_iter().collect(), + ); + inet_gw_map.insert( + g1_cfg.ipv4_cfg().unwrap().external_ips.floating_ips[2].into(), + [custom_igw1, all_ips_igw].into_iter().collect(), + ); + + let new_v4_cfg = g1_cfg.ipv4_cfg().map(|v| v.external_ips.clone()); + + let req = oxide_vpc::api::SetExternalIpsReq { + port_name: g1.port.name().to_string(), + external_ips_v4: new_v4_cfg, + external_ips_v6: None, + + // Setting the inet GW mappings for each external IP + // enables the limiting we aim to test here. + inet_gw_map: Some(inet_gw_map), + }; + nat::set_nat_rules(&g1.cfg, &g1.port, req).unwrap(); + update!(g1, ["incr:epoch", "set:nat.rules.out=8"]); + + // Send an ICMP packet for each destination, and verify that the + // correct source IP is written in (or the packet is denied). + let ident = 7; + let seq_no = 777; + let data = b"reunion\0"; + + // Default route. + let mut pkt1 = gen_icmp_echo_req( + g1_cfg.guest_mac, + g1_cfg.gateway_mac, + g1_cfg.ipv4_cfg().unwrap().private_ip.into(), + "77.77.77.77".parse().unwrap(), + ident, + seq_no, + &data[..], + 1, + ); + let res = g1.port.process(Out, &mut pkt1, ActionMeta::new()).unwrap(); + assert!(matches!(res, ProcessResult::Modified)); + assert_eq!( + pkt1.meta().inner_ip4().unwrap().src, + g1_cfg.ipv4().external_ips.ephemeral_ip.unwrap() + ); + incr!( + g1, + [ + "firewall.flows.out, firewall.flows.in", + "nat.flows.out, nat.flows.in", + "stats.port.out_uft_miss, uft.out", + "stats.port.out_modified", + ] + ); + + // 1.1.1.0/24 + let mut pkt1 = gen_icmp_echo_req( + g1_cfg.guest_mac, + g1_cfg.gateway_mac, + g1_cfg.ipv4_cfg().unwrap().private_ip.into(), + "1.1.1.1".parse().unwrap(), + ident, + seq_no, + &data[..], + 1, + ); + let res = g1.port.process(Out, &mut pkt1, ActionMeta::new()).unwrap(); + assert!(matches!(res, ProcessResult::Modified)); + assert!(&g1_cfg.ipv4().external_ips.floating_ips[..2] + .contains(&pkt1.meta().inner_ip4().unwrap().src)); + incr!( + g1, + [ + "firewall.flows.out, firewall.flows.in", + "nat.flows.out, nat.flows.in", + "stats.port.out_uft_miss, uft.out", + "stats.port.out_modified", + ] + ); + + // 2.2.2.0/24 + let mut pkt1 = gen_icmp_echo_req( + g1_cfg.guest_mac, + g1_cfg.gateway_mac, + g1_cfg.ipv4_cfg().unwrap().private_ip.into(), + "2.2.2.1".parse().unwrap(), + ident, + seq_no, + &data[..], + 1, + ); + let res = g1.port.process(Out, &mut pkt1, ActionMeta::new()).unwrap(); + assert!(matches!(res, ProcessResult::Modified)); + assert_eq!( + pkt1.meta().inner_ip4().unwrap().src, + g1_cfg.ipv4().external_ips.floating_ips[2] + ); + incr!( + g1, + [ + "firewall.flows.out, firewall.flows.in", + "nat.flows.out, nat.flows.in", + "stats.port.out_uft_miss, uft.out", + "stats.port.out_modified", + ] + ); + + // 3.3.3.0/24 + let mut pkt1 = gen_icmp_echo_req( + g1_cfg.guest_mac, + g1_cfg.gateway_mac, + g1_cfg.ipv4_cfg().unwrap().private_ip.into(), + "3.3.3.1".parse().unwrap(), + ident, + seq_no, + &data[..], + 1, + ); + let res = g1.port.process(Out, &mut pkt1, ActionMeta::new()).unwrap(); + assert!(matches!(res, ProcessResult::Drop { .. })); + incr!( + g1, + [ + "firewall.flows.out, firewall.flows.in", + "stats.port.out_uft_miss", + "stats.port.out_drop, stats.port.out_drop_layer", + ] + ); + + // 4.4.4.0/24 + let mut pkt1 = gen_icmp_echo_req( + g1_cfg.guest_mac, + g1_cfg.gateway_mac, + g1_cfg.ipv4_cfg().unwrap().private_ip.into(), + "4.4.4.1".parse().unwrap(), + ident, + seq_no, + &data[..], + 1, + ); + let res = g1.port.process(Out, &mut pkt1, ActionMeta::new()).unwrap(); + assert!(matches!(res, ProcessResult::Modified)); + assert!(&g1_cfg.ipv4().external_ips.floating_ips[..] + .contains(&pkt1.meta().inner_ip4().unwrap().src)); + incr!( + g1, + [ + "firewall.flows.out, firewall.flows.in", + "nat.flows.out, nat.flows.in", + "stats.port.out_uft_miss, uft.out", + "stats.port.out_modified", + ] + ); +}