From 3042e76571b262567cde4e299f8267da394bc809 Mon Sep 17 00:00:00 2001
From: Kyle Simpson <kyle@oxide.computer>
Date: Sat, 14 Dec 2024 01:06:35 +0000
Subject: [PATCH] Playing nice, again.

---
 lib/opte/src/ddi/mblk.rs    |  4 ++--
 lib/opte/src/engine/port.rs |  8 ++++----
 xde/src/mac/sys.rs          |  8 ++++----
 xde/src/xde.rs              | 21 ++++++++++++++++++---
 4 files changed, 28 insertions(+), 13 deletions(-)

diff --git a/lib/opte/src/ddi/mblk.rs b/lib/opte/src/ddi/mblk.rs
index 75b591d2..7877a81f 100644
--- a/lib/opte/src/ddi/mblk.rs
+++ b/lib/opte/src/ddi/mblk.rs
@@ -723,13 +723,13 @@ impl MsgBlk {
         #[cfg(all(not(feature = "std"), not(test)))]
         unsafe {
             // illumos_sys_hdrs::mac_hcksum_set(
-            //     self.inner.as_ptr(),
+            //     self.0.as_ptr(),
             //     0,0,0,0,
             //     ckflags,
             // );
             // if is_tcp {
             //     illumos_sys_hdrs::lso_info_set(
-            //         self.inner.as_ptr(),
+            //         self.0.as_ptr(),
             //         mss,
             //         0x10,
             //     );
diff --git a/lib/opte/src/engine/port.rs b/lib/opte/src/engine/port.rs
index c0c4f9b8..1ce85557 100644
--- a/lib/opte/src/engine/port.rs
+++ b/lib/opte/src/engine/port.rs
@@ -2322,7 +2322,7 @@ impl<N: NetworkImpl> Port<N> {
 
         let ufid_out = pkt.flow().mirror();
         let mut hte = UftEntry {
-            pair: KMutex::new(Some(ufid_out), KMutexType::Spin),
+            pair: KMutex::new(Some(ufid_out), KMutexType::Driver),
             xforms: xforms.compile(pkt.checksums_dirty()),
             epoch,
             l4_hash: ufid_in.crc32(),
@@ -2547,7 +2547,7 @@ impl<N: NetworkImpl> Port<N> {
         let res = self.layers_process(data, Out, pkt, &mut xforms, ameta);
 
         let hte = UftEntry {
-            pair: KMutex::new(None, KMutexType::Spin),
+            pair: KMutex::new(None, KMutexType::Driver),
             xforms: xforms.compile(pkt.checksums_dirty()),
             epoch,
             l4_hash: flow_before.crc32(),
@@ -2855,7 +2855,7 @@ impl TcpFlowEntryState {
                     bytes_in,
                     bytes_out: 0,
                 },
-                KMutexType::Spin,
+                KMutexType::Driver,
             ),
         }
     }
@@ -2876,7 +2876,7 @@ impl TcpFlowEntryState {
                     bytes_in: 0,
                     bytes_out,
                 },
-                KMutexType::Spin,
+                KMutexType::Driver,
             ),
         }
     }
diff --git a/xde/src/mac/sys.rs b/xde/src/mac/sys.rs
index 1a055978..d50f91b7 100644
--- a/xde/src/mac/sys.rs
+++ b/xde/src/mac/sys.rs
@@ -195,19 +195,19 @@ extern "C" {
 }
 
 #[repr(C)]
-#[derive(Clone, Copy, Default)]
+#[derive(Clone, Copy, Default, Debug)]
 pub struct lso_basic_tcp_ipv4_t {
     pub lso_max: t_uscalar_t,
 }
 
 #[repr(C)]
-#[derive(Clone, Copy, Default)]
+#[derive(Clone, Copy, Default, Debug)]
 pub struct lso_basic_tcp_ipv6_t {
     pub lso_max: t_uscalar_t,
 }
 
 #[repr(C)]
-#[derive(Clone, Copy, Default)]
+#[derive(Clone, Copy, Default, Debug)]
 pub struct lso_tunnel_tcp_ipv4_t {
     pub lso_max: t_uscalar_t,
     pub encap_max: t_uscalar_t,
@@ -215,7 +215,7 @@ pub struct lso_tunnel_tcp_ipv4_t {
 }
 
 #[repr(C)]
-#[derive(Clone, Copy, Default)]
+#[derive(Clone, Copy, Default, Debug)]
 pub struct lso_tunnel_tcp_ipv6_t {
     pub lso_max: t_uscalar_t,
     pub encap_max: t_uscalar_t,
diff --git a/xde/src/xde.rs b/xde/src/xde.rs
index 35b722e2..c857b7ed 100644
--- a/xde/src/xde.rs
+++ b/xde/src/xde.rs
@@ -239,7 +239,7 @@ pub struct xde_underlay_port {
     stream: Arc<DlsStream>,
 }
 
-#[derive(Copy, Clone)]
+#[derive(Copy, Clone, Debug)]
 struct OffloadInfo {
     lso_flags: u32,
     cso_flags: u32,
@@ -303,9 +303,9 @@ impl OffloadInfo {
                 TunnelTcpLsoFlags::from_bits_truncate(self.tun_v6_state.flags);
 
             tun_flags.contains(
+                // Inner v6 not yet supported, like.
                 TunnelTcpLsoFlags::GENEVE
-                    | TunnelTcpLsoFlags::INNER_IPV4
-                    | TunnelTcpLsoFlags::INNER_IPV6,
+                    | TunnelTcpLsoFlags::INNER_IPV4,
             )
         } else {
             false
@@ -1263,6 +1263,10 @@ unsafe fn init_underlay_ingress_handlers(
 ) -> Result<UnderlayState, OpteError> {
     let (u1, i1) = create_underlay_port(u1_name, "xdeu0")?;
     let (u2, i2) = create_underlay_port(u2_name, "xdeu1")?;
+    warn!(
+        "I'm seeing\ni1: {:?},\ni2: {:?}",
+        i1, i2
+    );
     Ok(UnderlayState { u1: u1.into(), u2: u2.into(), shared_props: i1 & i2 })
 }
 
@@ -1826,6 +1830,8 @@ unsafe fn xde_mc_tx_one(src_dev: &XdeDev, mut pkt: MsgBlk) -> *mut mblk_t {
                 return ptr::null_mut();
             }
 
+            // TODO: should these not just be copied from the original input
+            //       mblk?
             let cso_possible = src_dev.underlay_capab.should_request_cso();
             let lso_possible = src_dev.underlay_capab.should_request_lso();
 
@@ -1835,9 +1841,11 @@ unsafe fn xde_mc_tx_one(src_dev: &XdeDev, mut pkt: MsgBlk) -> *mut mblk_t {
                 // Boost MSS to use full jumbo frames if we know our path
                 // can be served purely on internal links.
                 // Recall that SDU does not include L2 size, hence 'non_eth_payl'
+
                 let mss = src_dev.underlay_capab.mtu
                     - 70
                     - (non_eth_payl_bytes as u32);
+                // let mss = 1448;
 
                 out_pkt.request_offload(is_tcp && lso_possible, mss);
 
@@ -1984,6 +1992,13 @@ unsafe extern "C" fn xde_mc_getcapab(
             let capab = capb_data as *mut mac_capab_lso_t;
             let desired_lso = shared_underlay_caps.upstream_lso();
 
+            warn!(
+                "I'm advertising flags {:x?}, v4 {:?} v6 {:?}",
+                desired_lso.lso_flags,
+                desired_lso.lso_basic_tcp_ipv4,
+                desired_lso.lso_basic_tcp_ipv6
+            );
+
             unsafe {
                 // Don't write the newer capabs -- don't want to corrupt
                 // memory on older illumos and/or CI.