Tunneled LSO/CSO Support

crates/illumos-sys-hdrs/Cargo.toml

@@ -8,4 +8,7 @@ repository.workspace = true
 
 [features]
 default = []
-kernel = []
+kernel = []
+
+[dependencies]
+bitflags.workspace = true

crates/illumos-sys-hdrs/src/lib.rs

@@ -11,6 +11,10 @@
 pub mod kernel;
 #[cfg(feature = "kernel")]
 pub use kernel::*;
+use mac::mac_ether_offload_info_t;
+use mac::mac_ether_tun_info_t;
+
+pub mod mac;
 
 use core::ptr;
 
@@ -236,6 +240,9 @@ pub struct dblk_t {
     pub db_struioun: u64,        // imprecise
     pub db_fthdr: *const c_void, // imprecise
     pub db_credp: *const c_void, // imprecise
+
+    pub db_meoi: mac_ether_offload_info_t,
+    pub db_mett: mac_ether_tun_info_t,
 }
 
 impl Default for dblk_t {
@@ -259,6 +266,9 @@ impl Default for dblk_t {
             db_struioun: 0,
             db_fthdr: ptr::null(),
             db_credp: ptr::null(),
+
+            db_meoi: Default::default(),
+            db_mett: Default::default(),
         }
     }
 }

crates/illumos-sys-hdrs/src/mac.rs

@@ -0,0 +1,123 @@
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this
+// file, You can obtain one at https://mozilla.org/MPL/2.0/.
+
+// Copyright 2024 Oxide Computer Company
+
+#[cfg(feature = "kernel")]
+use crate::mblk_t;
+use bitflags::bitflags;
+
+// ======================================================================
+// uts/common/sys/mac_provider.h
+// ======================================================================
+
+bitflags! {
+#[repr(C)]
+#[derive(Clone, Copy, Debug, Default)]
+/// Flags which denote the valid fields of a `mac_ether_offload_info_t`
+/// or `mac_ether_tun_info_t`.
+pub struct MacEtherOffloadFlags: u8 {
+    /// `l2hlen` and `l3proto` are set.
+    const L2INFO_SET     = 1 << 0;
+    /// The ethernet header contains a VLAN tag.
+    const VLAN_TAGGED    = 1 << 1;
+    /// `l3hlen` and `l4proto` are set.
+    const L3INFO_SET     = 1 << 2;
+    /// `l4hlen` is set.
+    const L4INFO_SET     = 1 << 3;
+    /// `tuntype` is set.
+    const TUNINFO_SET    = 1 << 4;
+}
+}
+
+#[repr(C)]
+#[derive(Copy, Clone, Debug, Default, Eq, PartialEq)]
+pub struct MacTunType(u8);
+
+impl MacTunType {
+    pub const NONE: Self = Self(0);
+    pub const GENEVE: Self = Self(1);
+    pub const VXLAN: Self = Self(2);
+}
+
+#[repr(C)]
+#[derive(Clone, Copy, Debug, Default)]
+pub struct mac_ether_offload_info_t {
+    pub meoi_flags: MacEtherOffloadFlags,
+    pub meoi_l2hlen: u8,
+    pub meoi_l3proto: u16,
+    pub meoi_l3hlen: u16,
+    pub meoi_l4proto: u8,
+    pub meoi_l4hlen: u8,
+    pub meoi_len: u32,
+}
+
+#[repr(C)]
+#[derive(Clone, Copy, Debug, Default)]
+pub struct mac_ether_tun_info_t {
+    pub mett_flags: MacEtherOffloadFlags,
+    pub mett_tuntype: MacTunType,
+    pub mett_l2hlen: u8,
+    pub mett_l3proto: u16,
+    pub mett_l3hlen: u16,
+}
+
+#[cfg(feature = "kernel")]
+extern "C" {
+    pub fn lso_info_set(mp: *mut mblk_t, mss: u32, flags: u32);
+    pub fn lso_info_cleanup(mp: *mut mblk_t);
+    pub fn mac_hcksum_set(
+        mp: *mut mblk_t,
+        start: u32,
+        stuff: u32,
+        end: u32,
+        value: u32,
+        flags: u32,
+    );
+    pub fn mac_hcksum_get(
+        mp: *mut mblk_t,
+        start: *mut u32,
+        stuff: *mut u32,
+        end: *mut u32,
+        value: *mut u32,
+        flags: *mut u32,
+    );
+}
+
+// ======================================================================
+// uts/common/sys/pattr.h
+// ======================================================================
+
+bitflags! {
+/// Flags which denote checksum and LSO state for an `mblk_t`.
+pub struct MblkOffloadFlags: u16 {
+    /// Tx: IPv4 header checksum must be computer by hardware.
+    const HCK_IPV4_HDRCKSUM = 1 << 0;
+    /// Rx: IPv4 header checksum was verified correct by hardware.
+    const HCK_IPV4_HDRCKSUM_OK = Self::HCK_IPV4_HDRCKSUM.bits();
+    /// * Tx: Compute partial checksum based on start/stuff/end offsets.
+    /// * Rx: Partial checksum computed and attached.
+    const HCK_PARTIALCKSUM = 1 << 1;
+    /// * Tx: Compute full (pseudo + l4 + payload) cksum for this packet.
+    /// * Rx: Full checksum was computed in hardware, and is attached.
+    const HCK_FULLCKSUM = 1 << 2;
+    /// Rx: Hardware has verified that L3/L4 checksums are correct.
+    const HCK_FULLCKSUM_OK = 1 << 3;
+    /// Tx: Hardware must perform LSO.
+    const HW_LSO = 1 << 4;
+    /// Tx: Hardware must compute all checksum for the outer tunnel
+    /// encapsulation of this packet.
+    const HCK_FULLOUTERCKSUM = 1 << 5;
+
+    const HCK_FLAGS = Self::HCK_IPV4_HDRCKSUM.bits() |
+        Self::HCK_PARTIALCKSUM.bits() | Self::HCK_FULLCKSUM.bits() |
+        Self::HCK_FULLCKSUM_OK.bits() | Self::HCK_FULLOUTERCKSUM.bits();
+
+    const HCK_TX_FLAGS = Self::HCK_IPV4_HDRCKSUM.bits() |
+        Self::HCK_PARTIALCKSUM.bits() | Self::HCK_FULLCKSUM.bits() |
+        Self::HCK_FULLOUTERCKSUM.bits();
+
+    const HW_LSO_FLAGS = Self::HW_LSO.bits();
+}
+}

lib/opte/Cargo.toml

@@ -29,7 +29,7 @@ opte-api.workspace = true
 
 ingot.workspace = true
 
-bitflags.workspace = true
+bitflags = { workspace = true , features = ["serde"] }
 cfg-if.workspace = true
 crc32fast = { workspace = true, optional = true }
 dyn-clone.workspace = true

lib/opte/src/ddi/mblk.rs

@@ -27,6 +27,12 @@ use illumos_sys_hdrs as ddi;
 use illumos_sys_hdrs::c_uchar;
 #[cfg(any(feature = "std", test))]
 use illumos_sys_hdrs::dblk_t;
+use illumos_sys_hdrs::mac::mac_ether_offload_info_t;
+use illumos_sys_hdrs::mac::mac_ether_tun_info_t;
+#[cfg(all(not(feature = "std"), not(test)))]
+use illumos_sys_hdrs::mac::MacEtherOffloadFlags;
+use illumos_sys_hdrs::mac::MacTunType;
+use illumos_sys_hdrs::mac::MblkOffloadFlags;
 use illumos_sys_hdrs::mblk_t;
 use illumos_sys_hdrs::uintptr_t;
 use ingot::types::Emit;
@@ -36,6 +42,14 @@ use ingot::types::Read;
 
 pub static MBLK_MAX_SIZE: usize = u16::MAX as usize;
 
+/// Abstractions over an `mblk_t` which can be returned to their
+/// raw pointer representation.
+pub trait AsMblk {
+    /// Consume `self`, returning the underlying `mblk_t`. The caller of this
+    /// function now owns the underlying segment chain.
+    fn unwrap_mblk(self) -> Option<NonNull<mblk_t>>;
+}
+
 /// The head and tail of an mblk_t list.
 struct MsgBlkChainInner {
     head: NonNull<mblk_t>,
@@ -146,11 +160,10 @@ impl MsgBlkChain {
             self.0 = Some(MsgBlkChainInner { head: pkt, tail: pkt });
         }
     }
+}
 
-    /// Return the head of the underlying `mblk_t` packet chain and
-    /// consume `self`. The caller of this function now owns the
-    /// `mblk_t` segment chain.
-    pub fn unwrap_mblk(mut self) -> Option<NonNull<mblk_t>> {
+impl AsMblk for MsgBlkChain {
+    fn unwrap_mblk(mut self) -> Option<NonNull<mblk_t>> {
         self.0.take().map(|v| v.head)
     }
 }
@@ -615,9 +628,7 @@ impl MsgBlk {
     /// consume `self`. The caller of this function now owns the
     /// `mblk_t` segment chain.
     pub fn unwrap_mblk(self) -> NonNull<mblk_t> {
-        let ptr_out = self.0;
-        _ = ManuallyDrop::new(self);
-        ptr_out
+        AsMblk::unwrap_mblk(self).unwrap()
     }
 
     /// Wrap the `mblk_t` packet in a [`MsgBlk`], taking ownership of
@@ -707,6 +718,108 @@ impl MsgBlk {
 
         self.0 = head;
     }
+
+    /// Copies the offload information from this message block to
+    /// another, including checksum/LSO flags and TCP MSS (if set).
+    pub fn copy_offload_info_to(&self, other: &mut Self) {
+        unsafe {
+            let info = offload_info(self.0);
+            set_offload_info(other.0, info);
+        }
+    }
+
+    #[allow(unused)]
+    pub fn request_offload(
+        &mut self,
+        cksum_needed: bool,
+        is_tcp: bool,
+        mss: u32,
+    ) {
+        let ckflags = if cksum_needed {
+            MblkOffloadFlags::HCK_IPV4_HDRCKSUM
+                | MblkOffloadFlags::HCK_FULLCKSUM
+        } else {
+            MblkOffloadFlags::empty()
+        };
+        #[cfg(all(not(feature = "std"), not(test)))]
+        unsafe {
+            if !ckflags.is_empty() {
+                illumos_sys_hdrs::mac::mac_hcksum_set(
+                    self.0.as_ptr(),
+                    0,
+                    0,
+                    0,
+                    0,
+                    ckflags.bits() as u32,
+                );
+            }
+            if is_tcp {
+                illumos_sys_hdrs::mac::lso_info_set(
+                    self.0.as_ptr(),
+                    mss,
+                    MblkOffloadFlags::HW_LSO.bits() as u32,
+                );
+            }
+        }
+    }
+
+    #[allow(unused)]
+    pub fn strip_lso(&mut self) {
+        #[cfg(all(not(feature = "std"), not(test)))]
+        unsafe {
+            illumos_sys_hdrs::mac::lso_info_cleanup(self.0.as_ptr());
+        }
+    }
+
+    #[allow(unused)]
+    pub fn set_tuntype(&mut self, tuntype: MacTunType) {
+        #[cfg(all(not(feature = "std"), not(test)))]
+        unsafe {
+            (*(*self.0.as_ptr()).b_datap).db_mett.mett_tuntype = tuntype;
+            (*(*self.0.as_ptr()).b_datap).db_mett.mett_flags |=
+                MacEtherOffloadFlags::TUNINFO_SET;
+        }
+    }
+
+    #[allow(unused)]
+    pub fn fill_offload_info(
+        &mut self,
+        tun_meoi: &mac_ether_tun_info_t,
+        ulp_meoi: &mac_ether_offload_info_t,
+    ) {
+        #[cfg(all(not(feature = "std"), not(test)))]
+        unsafe {
+            (*(*self.0.as_ptr()).b_datap).db_mett = *tun_meoi;
+            (*(*self.0.as_ptr()).b_datap).db_meoi = *ulp_meoi;
+        }
+    }
+
+    #[allow(unused)]
+    pub fn cksum_flags(&self) -> MblkOffloadFlags {
+        let mut out = 0u32;
+
+        #[cfg(all(not(feature = "std"), not(test)))]
+        unsafe {
+            illumos_sys_hdrs::mac::mac_hcksum_get(
+                self.0.as_ptr(),
+                ptr::null_mut(),
+                ptr::null_mut(),
+                ptr::null_mut(),
+                ptr::null_mut(),
+                &raw mut out,
+            )
+        };
+
+        MblkOffloadFlags::from_bits_retain(out as u16)
+    }
+}
+
+impl AsMblk for MsgBlk {
+    fn unwrap_mblk(self) -> Option<NonNull<mblk_t>> {
+        let ptr_out = self.0;
+        _ = ManuallyDrop::new(self);
+        Some(ptr_out)
+    }
 }
 
 /// An interior node of an [`MsgBlk`]'s chain, accessed via iterator.
@@ -1046,6 +1159,8 @@ pub fn mock_desballoc(buf: Vec<u8>) -> *mut mblk_t {
         db_struioun: 0,
         db_fthdr: ptr::null(),
         db_credp: ptr::null(),
+
+        ..Default::default()
     });
 
     let dbp = Box::into_raw(dblk);