VPC Subnet Routing

bench/src/packet.rs

@@ -295,6 +295,7 @@ impl BenchPacketInstance for UlpProcessInstance {
  &g1.port,
  IpCidr::Ip4("0.0.0.0/0".parse().unwrap()),
  RouterTarget::InternetGateway,
+ RouterClass::System,
  )
  .unwrap();
  incr!(g1, ["epoch", "router.rules.out"]);
@@ -303,6 +304,7 @@ impl BenchPacketInstance for UlpProcessInstance {
  &g1.port,
  IpCidr::Ip6("::/0".parse().unwrap()),
  RouterTarget::InternetGateway,
+ RouterClass::System,
  )
  .unwrap();
  incr!(g1, ["epoch", "router.rules.out"]);

bin/opteadm/src/bin/opteadm.rs

@@ -2,7 +2,7 @@
 // License, v. 2.0. If a copy of the MPL was not distributed with this
 // file, You can obtain one at https://mozilla.org/MPL/2.0/.
 
-// Copyright 2023 Oxide Computer Company
+// Copyright 2024 Oxide Computer Company
 
 use clap::Args;
 use clap::Parser;
@@ -26,6 +26,8 @@ use oxide_vpc::api::AddRouterEntryReq;
 use oxide_vpc::api::Address;
 use oxide_vpc::api::ClearVirt2BoundaryReq;
 use oxide_vpc::api::ClearVirt2PhysReq;
+use oxide_vpc::api::DelRouterEntryReq;
+use oxide_vpc::api::DelRouterEntryResp;
 use oxide_vpc::api::DhcpCfg;
 use oxide_vpc::api::ExternalIpCfg;
 use oxide_vpc::api::Filters as FirewallFilters;
@@ -39,6 +41,8 @@ use oxide_vpc::api::PortInfo;
 use oxide_vpc::api::Ports;
 use oxide_vpc::api::ProtoFilter;
 use oxide_vpc::api::RemFwRuleReq;
+use oxide_vpc::api::RemoveCidrResp;
+use oxide_vpc::api::RouterClass;
 use oxide_vpc::api::RouterTarget;
 use oxide_vpc::api::SNat4Cfg;
 use oxide_vpc::api::SNat6Cfg;
@@ -220,24 +224,45 @@ enum Command {
 
  /// Add a new router entry, either IPv4 or IPv6.
  AddRouterEntry {
- /// The OPTE port to which the route is added
- #[arg(short)]
- port: String,
- /// The network destination to which the route applies.
- dest: IpCidr,
- /// The location to which traffic matching the destination is sent.
- target: RouterTarget,
+ #[command(flatten)]
+ route: RouterRule,
+ },
+
+ /// Remove an existing router entry, either IPv4 or IPv6.
+ DelRouterEntry {
+ #[command(flatten)]
+ route: RouterRule,
  },
 
  /// Configure external network addresses used by a port for VPC-external traffic.
  SetExternalIps {
- /// The OPTE port to which the route is added
+ /// The OPTE port to configure
  #[arg(short)]
  port: String,
 
  #[command(flatten)]
  external_net: ExternalNetConfig,
  },
+
+ /// Allows a guest to send and receive traffic on a given CIDR block.
+ AllowCidr {
+ /// The OPTE port to configure
+ #[arg(short)]
+ port: String,
+
+ /// The IP block to allow through the gateway.
+ prefix: IpCidr,
+ },
+
+ /// Prevents a guest from sending/receiving traffic on a given CIDR block.
+ RemoveCidr {
+ /// The OPTE port to configure
+ #[arg(short)]
+ port: String,
+
+ /// The IP block to deny at the gateway.
+ prefix: IpCidr,
+ },
 }
 
 #[derive(Debug, Parser)]
@@ -265,6 +290,19 @@ impl From<Filters> for FirewallFilters {
  }
 }
 
+#[derive(Debug, Parser)]
+struct RouterRule {
+ /// The OPTE port to which the route change is applied.
+ #[arg(short)]
+ port: String,
+ /// The network destination to which the route applies.
+ dest: IpCidr,
+ /// The location to which traffic matching the destination is sent.
+ target: RouterTarget,
+ /// The class of router a rule belongs to ('system' or 'custom')
+ class: RouterClass,
+}
+
 // TODO: expand this to allow for v4 and v6 simultaneously?
 /// Per-port configuration for rack-external networking.
 #[derive(Args, Clone, Debug)]
@@ -705,11 +743,26 @@ fn main() -> anyhow::Result<()> {
  hdl.clear_v2b(&req)?;
  }
 
- Command::AddRouterEntry { port, dest, target } => {
- let req = AddRouterEntryReq { port_name: port, dest, target };
+ Command::AddRouterEntry {
+ route: RouterRule { port, dest, target, class },
+ } => {
+ let req =
+ AddRouterEntryReq { port_name: port, dest, target, class };
  hdl.add_router_entry(&req)?;
  }
 
+ Command::DelRouterEntry {
+ route: RouterRule { port, dest, target, class },
+ } => {
+ let req =
+ DelRouterEntryReq { port_name: port, dest, target, class };
+ if let DelRouterEntryResp::NotFound = hdl.del_router_entry(&req)? {
+ anyhow::bail!(
+ "could not delete entry -- no matching rule found"
+ );
+ }
+ }
+
  Command::SetExternalIps { port, external_net } => {
  if let Ok(cfg) =
  ExternalIpCfg::<Ipv4Addr>::try_from(external_net.clone())
@@ -734,6 +787,18 @@ fn main() -> anyhow::Result<()> {
  anyhow::bail!("expected IPv4 *or* IPv6 config.");
  }
  }
+
+ Command::AllowCidr { port, prefix } => {
+ hdl.allow_cidr(&port, prefix)?;
+ }
+
+ Command::RemoveCidr { port, prefix } => {
+ if let RemoveCidrResp::NotFound = hdl.remove_cidr(&port, prefix)? {
+ anyhow::bail!(
+ "could not remove cidr {prefix} from gateway -- not found"
+ );
+ }
+ }
  }
 
  Ok(())

bin/opteadm/src/lib.rs

@@ -2,11 +2,12 @@
 // License, v. 2.0. If a copy of the MPL was not distributed with this
 // file, You can obtain one at https://mozilla.org/MPL/2.0/.
 
-// Copyright 2023 Oxide Computer Company
+// Copyright 2024 Oxide Computer Company
 
 //! OPTE driver administration library
 
 use opte::api::ClearXdeUnderlayReq;
+use opte::api::IpCidr;
 use opte::api::NoResp;
 use opte::api::OpteCmd;
 use opte::api::SetXdeUnderlayReq;
@@ -15,9 +16,12 @@ use opte_ioctl::run_cmd_ioctl;
 use opte_ioctl::Error;
 use oxide_vpc::api::AddFwRuleReq;
 use oxide_vpc::api::AddRouterEntryReq;
+use oxide_vpc::api::AllowCidrReq;
 use oxide_vpc::api::ClearVirt2BoundaryReq;
 use oxide_vpc::api::ClearVirt2PhysReq;
 use oxide_vpc::api::CreateXdeReq;
+use oxide_vpc::api::DelRouterEntryReq;
+use oxide_vpc::api::DelRouterEntryResp;
 use oxide_vpc::api::DeleteXdeReq;
 use oxide_vpc::api::DhcpCfg;
 use oxide_vpc::api::DumpVirt2BoundaryReq;
@@ -27,6 +31,8 @@ use oxide_vpc::api::DumpVirt2PhysResp;
 use oxide_vpc::api::FirewallRule;
 use oxide_vpc::api::ListPortsResp;
 use oxide_vpc::api::RemFwRuleReq;
+use oxide_vpc::api::RemoveCidrReq;
+use oxide_vpc::api::RemoveCidrResp;
 use oxide_vpc::api::SetExternalIpsReq;
 use oxide_vpc::api::SetFwRulesReq;
 use oxide_vpc::api::SetVirt2BoundaryReq;
@@ -276,11 +282,45 @@ impl OpteAdm {
  run_cmd_ioctl(self.device.as_raw_fd(), cmd, Some(&req))
  }
 
+ pub fn del_router_entry(
+ &self,
+ req: &DelRouterEntryReq,
+ ) -> Result<DelRouterEntryResp, Error> {
+ let cmd = OpteCmd::DelRouterEntry;
+ run_cmd_ioctl(self.device.as_raw_fd(), cmd, Some(&req))
+ }
+
  pub fn set_external_ips(
  &self,
  req: &SetExternalIpsReq,
  ) -> Result<NoResp, Error> {
  let cmd = OpteCmd::SetExternalIps;
  run_cmd_ioctl(self.device.as_raw_fd(), cmd, Some(&req))
  }
+
+ pub fn allow_cidr(
+ &self,
+ port_name: &str,
+ cidr: IpCidr,
+ ) -> Result<NoResp, Error> {
+ let cmd = OpteCmd::AllowCidr;
+ run_cmd_ioctl(
+ self.device.as_raw_fd(),
+ cmd,
+ Some(&AllowCidrReq { cidr, port_name: port_name.into() }),
+ )
+ }
+
+ pub fn remove_cidr(
+ &self,
+ port_name: &str,
+ cidr: IpCidr,
+ ) -> Result<RemoveCidrResp, Error> {
+ let cmd = OpteCmd::RemoveCidr;
+ run_cmd_ioctl(
+ self.device.as_raw_fd(),
+ cmd,
+ Some(&RemoveCidrReq { cidr, port_name: port_name.into() }),
+ )
+ }
 }

crates/opte-api/src/cmd.rs

@@ -2,7 +2,7 @@
 // License, v. 2.0. If a copy of the MPL was not distributed with this
 // file, You can obtain one at https://mozilla.org/MPL/2.0/.
 
-// Copyright 2023 Oxide Computer Company
+// Copyright 2024 Oxide Computer Company
 
 use super::encap::Vni;
 use super::ip::IpCidr;
@@ -37,11 +37,14 @@ pub enum OpteCmd {
  DumpVirt2Boundary = 54, // dump the v2b mappings
  ClearVirt2Phys = 55, // clear a v2p mapping
  AddRouterEntry = 60, // add a router entry for IP dest
+ DelRouterEntry = 61, // remove a router entry for IP dest
  CreateXde = 70, // create a new xde device
  DeleteXde = 71, // delete an xde device
  SetXdeUnderlay = 72, // set xde underlay devices
  ClearXdeUnderlay = 73, // clear xde underlay devices
  SetExternalIps = 80, // set xde external IPs for a port
+ AllowCidr = 90, // allow ip block through gateway tx/rx
+ RemoveCidr = 91, // deny ip block through gateway tx/rx
 }
 
 impl TryFrom<c_int> for OpteCmd {
@@ -66,11 +69,14 @@ impl TryFrom<c_int> for OpteCmd {
  54 => Ok(Self::DumpVirt2Boundary),
  55 => Ok(Self::ClearVirt2Phys),
  60 => Ok(Self::AddRouterEntry),
+ 61 => Ok(Self::DelRouterEntry),
  70 => Ok(Self::CreateXde),
  71 => Ok(Self::DeleteXde),
  72 => Ok(Self::SetXdeUnderlay),
  73 => Ok(Self::ClearXdeUnderlay),
  80 => Ok(Self::SetExternalIps),
+ 90 => Ok(Self::AllowCidr),
+ 91 => Ok(Self::RemoveCidr),
  _ => Err(()),
  }
  }

crates/opte-api/src/lib.rs

@@ -2,7 +2,7 @@
 // License, v. 2.0. If a copy of the MPL was not distributed with this
 // file, You can obtain one at https://mozilla.org/MPL/2.0/.
 
-// Copyright 2023 Oxide Computer Company
+// Copyright 2024 Oxide Computer Company
 
 #![no_std]
 #![deny(unreachable_patterns)]
@@ -47,7 +47,7 @@ pub use ulp::*;
 ///
 /// We rely on CI and the check-api-version.sh script to verify that
 /// this number is incremented anytime the oxide-api code changes.
-pub const API_VERSION: u64 = 31;
+pub const API_VERSION: u64 = 32;
 
 /// Major version of the OPTE package.
 pub const MAJOR_VERSION: u64 = 0;

lib/opte-ioctl/src/lib.rs

@@ -2,7 +2,7 @@
 // License, v. 2.0. If a copy of the MPL was not distributed with this
 // file, You can obtain one at https://mozilla.org/MPL/2.0/.
 
-// Copyright 2022 Oxide Computer Company
+// Copyright 2024 Oxide Computer Company
 
 use opte::api::ClearXdeUnderlayReq;
 use opte::api::CmdOk;
@@ -14,14 +14,20 @@ use opte::api::SetXdeUnderlayReq;
 use opte::api::API_VERSION;
 use opte::api::XDE_IOC_OPTE_CMD;
 use oxide_vpc::api::AddRouterEntryReq;
+use oxide_vpc::api::AllowCidrReq;
 use oxide_vpc::api::ClearVirt2BoundaryReq;
 use oxide_vpc::api::ClearVirt2PhysReq;
 use oxide_vpc::api::CreateXdeReq;
+use oxide_vpc::api::DelRouterEntryReq;
+use oxide_vpc::api::DelRouterEntryResp;
 use oxide_vpc::api::DeleteXdeReq;
 use oxide_vpc::api::DhcpCfg;
 use oxide_vpc::api::DumpVirt2PhysReq;
 use oxide_vpc::api::DumpVirt2PhysResp;
+use oxide_vpc::api::IpCidr;
 use oxide_vpc::api::ListPortsResp;
+use oxide_vpc::api::RemoveCidrReq;
+use oxide_vpc::api::RemoveCidrResp;
 use oxide_vpc::api::SetExternalIpsReq;
 use oxide_vpc::api::SetFwRulesReq;
 use oxide_vpc::api::SetVirt2BoundaryReq;
@@ -203,6 +209,14 @@ impl OpteHdl {
  run_cmd_ioctl(self.device.as_raw_fd(), cmd, Some(&req))
  }
 
+ pub fn del_router_entry(
+ &self,
+ req: &DelRouterEntryReq,
+ ) -> Result<DelRouterEntryResp, Error> {
+ let cmd = OpteCmd::DelRouterEntry;
+ run_cmd_ioctl(self.device.as_raw_fd(), cmd, Some(&req))
+ }
+
  pub fn set_fw_rules(&self, req: &SetFwRulesReq) -> Result<NoResp, Error> {
  let cmd = OpteCmd::SetFwRules;
  run_cmd_ioctl(self.device.as_raw_fd(), cmd, Some(&req))
@@ -215,6 +229,32 @@ impl OpteHdl {
  let cmd = OpteCmd::SetExternalIps;
  run_cmd_ioctl(self.device.as_raw_fd(), cmd, Some(&req))
  }
+
+ pub fn allow_cidr(
+ &self,
+ port_name: &str,
+ cidr: IpCidr,
+ ) -> Result<NoResp, Error> {
+ let cmd = OpteCmd::AllowCidr;
+ run_cmd_ioctl(
+ self.device.as_raw_fd(),
+ cmd,
+ Some(&AllowCidrReq { cidr, port_name: port_name.into() }),
+ )
+ }
+
+ pub fn remove_cidr(
+ &self,
+ port_name: &str,
+ cidr: IpCidr,
+ ) -> Result<RemoveCidrResp, Error> {
+ let cmd = OpteCmd::RemoveCidr;
+ run_cmd_ioctl(
+ self.device.as_raw_fd(),
+ cmd,
+ Some(&RemoveCidrReq { cidr, port_name: port_name.into() }),
+ )
+ }
 }
 
 #[cfg(target_os = "illumos")]

lib/opte-test-utils/src/lib.rs

@@ -63,6 +63,7 @@ pub use oxide_vpc::api::IpCfg;
 pub use oxide_vpc::api::Ipv4Cfg;
 pub use oxide_vpc::api::Ipv6Cfg;
 pub use oxide_vpc::api::PhysNet;
+pub use oxide_vpc::api::RouterClass;
 pub use oxide_vpc::api::RouterTarget;
 pub use oxide_vpc::api::SNat4Cfg;
 pub use oxide_vpc::api::SNat6Cfg;
@@ -333,6 +334,7 @@ pub fn oxide_net_setup2(
  &port,
  IpCidr::Ip4(cfg.ipv4().vpc_subnet),
  RouterTarget::VpcSubnet(IpCidr::Ip4(cfg.ipv4().vpc_subnet)),
+ RouterClass::System,
  )
  .unwrap();