Cache routes received from next_hop (#499)

This PR inserts a caching layer in front of `next_hop` to store routes
for a given `(IpAddr6, L4Hash)` for around 100ms, and reorganises
routing-related code into `xde::route`. This is a necessary step as
today we spend around 21% of `xde_mc_tx` in route lookup on a
per-outbound-packet basis. As a consequence, we need install a new
`Periodic` to flush expired entries every 1s, as otherwise we can have a
large amountof detritus pileup from different L4Hash values.

Currently this is implemented as a per-port cache rather than shared
across the driver -- I've done this because it will give us better
sharding of concurrent readers/writers at higher port/guest counts. The
PR as-is pulls us up from ~1.74Gbps to 2Gbps on a single link between
sn9/14.

Author: FelixMcFelix

lib/opte/src/d_error.rs

@@ -145,7 +145,7 @@ impl<const L: usize> LabelBlock<L> {
     }
 
     /// Provides access to all stored [`CStr`]s.
-    pub fn entries<'a>(&'a self) -> LabelBlockIter<'a, L> {
+    pub fn entries(&self) -> LabelBlockIter<'_, L> {
         LabelBlockIter { pos: 0, inner: self }
     }
 

lib/opte/src/engine/packet.rs

@@ -469,7 +469,7 @@ impl PacketChain {
 
     /// Removes the next packet from the top of the chain and returns
     /// it, taking ownership.
-    pub fn next(&mut self) -> Option<Packet<Initialized>> {
+    pub fn pop_front(&mut self) -> Option<Packet<Initialized>> {
         if let Some(ref mut list) = &mut self.inner {
             unsafe {
                 let curr = list.head.as_ptr();
@@ -554,7 +554,7 @@ impl Drop for PacketChain {
                     unsafe { ddi::freemsgchain(list.head.as_ptr()) };
                 }
             } else {
-                while let Some(pkt) = self.next() {
+                while let Some(pkt) = self.pop_front() {
                     drop(pkt);
                 }
             }
@@ -3945,7 +3945,7 @@ mod test {
 
     fn create_linked_mblks(n: usize) -> Vec<*mut mblk_t> {
         let mut els = vec![];
-        for i in 0..n {
+        for _ in 0..n {
             els.push(allocb(8));
         }
 
@@ -3962,7 +3962,7 @@ mod test {
 
     #[test]
     fn chain_has_correct_ends() {
-        let mut els = create_linked_mblks(3);
+        let els = create_linked_mblks(3);
 
         let chain = unsafe { PacketChain::new(els[0]) }.unwrap();
         let chain_inner = chain.inner.as_ref().unwrap();
@@ -3976,7 +3976,7 @@ mod test {
 
         let mut chain = unsafe { PacketChain::new(els[0]) }.unwrap();
 
-        let p0 = chain.next().unwrap();
+        let p0 = chain.pop_front().unwrap();
         assert_eq!(p0.mblk_addr(), els[0] as uintptr_t);
         unsafe {
             assert!((*els[0]).b_prev.is_null());
@@ -4024,10 +4024,10 @@ mod test {
         let mut chain = unsafe { PacketChain::new(els[0]) }.unwrap();
 
         for i in 0..els.len() {
-            let pkt = chain.next().unwrap();
+            let pkt = chain.pop_front().unwrap();
             assert_eq!(pkt.mblk_addr(), els[i] as uintptr_t);
         }
 
-        assert!(chain.next().is_none());
+        assert!(chain.pop_front().is_none());
     }
 }

lib/opte/src/engine/rule.rs

@@ -834,7 +834,7 @@ impl<'a> Rule<Finalized> {
         #[cfg(debug_assertions)]
         {
             if let Some(preds) = &self.state.preds {
-                if preds.hdr_preds.len() == 0 && preds.data_preds.len() == 0 {
+                if preds.hdr_preds.is_empty() && preds.data_preds.is_empty() {
                     panic!(
                         "bug: RulePredicates must have at least one \
                             predicate"

xde/src/lib.rs

@@ -2,7 +2,7 @@
 // License, v. 2.0. If a copy of the MPL was not distributed with this
 // file, You can obtain one at https://mozilla.org/MPL/2.0/.
 
-// Copyright 2022 Oxide Computer Company
+// Copyright 2024 Oxide Computer Company
 
 // xde - A mac provider for OPTE-based network implementations.
 #![feature(extern_types)]
@@ -44,6 +44,7 @@ pub mod dls;
 pub mod ip;
 pub mod mac;
 mod mac_sys;
+pub mod route;
 pub mod secpolicy;
 pub mod sys;
 pub mod xde;