net: support serializing RxBuffer

Now, that we pre-process the buffers that guest provides for performing
RX, we need to save them in the VM state snapshot file, for networking
to work correctly post snapshot resume.

Implement Persist for RxBuffers and and plug them in the
(de)serialization logic of the network device.

Co-authored-by: Egor Lazarchuk <yegorlz@amazon.co.uk>
Signed-off-by: Babis Chalios <bchalios@amazon.es>

Author: bchalios

src/vmm/src/devices/virtio/iovec.rs

@@ -4,6 +4,7 @@
 use std::io::ErrorKind;
 
 use libc::{c_void, iovec, size_t};
+use serde::{Deserialize, Serialize};
 #[cfg(not(kani))]
 use smallvec::SmallVec;
 use vm_memory::bitmap::Bitmap;
@@ -218,7 +219,7 @@ impl IoVecBuffer {
     }
 }
 
-#[derive(Debug)]
+#[derive(Debug, Clone, Serialize, Deserialize)]
 pub struct ParsedDescriptorChain {
     pub head_index: u16,
     pub length: u32,
@@ -233,9 +234,9 @@ pub struct ParsedDescriptorChain {
 #[derive(Debug)]
 pub struct IoVecBufferMut {
     // container of the memory regions included in this IO vector
-    vecs: IovDeque,
+    pub vecs: IovDeque,
     // Total length of the IoVecBufferMut
-    len: usize,
+    pub len: usize,
 }
 
 impl IoVecBufferMut {

src/vmm/src/devices/virtio/net/device.rs

@@ -99,16 +99,16 @@ enum AddRxBufferError {
 
 /// A map of all the memory the guest has provided us with for performing RX
 #[derive(Debug)]
-pub(crate) struct RxBuffers {
+pub struct RxBuffers {
     // minimum size of a usable buffer for doing RX
-    min_buffer_size: u32,
+    pub min_buffer_size: u32,
     // An [`IoVecBufferMut`] covering all the memory we have available for receiving network
     // frames.
-    iovec: IoVecBufferMut,
+    pub iovec: IoVecBufferMut,
     // A map of which part of the memory belongs to which `DescriptorChain` object
-    parsed_descriptors: VecDeque<ParsedDescriptorChain>,
+    pub parsed_descriptors: VecDeque<ParsedDescriptorChain>,
     // Buffers that we have used and they are ready to be given back to the guest.
-    deferred_descriptor: Option<ParsedDescriptorChain>,
+    pub deferred_descriptor: Option<ParsedDescriptorChain>,
 }
 
 impl RxBuffers {
@@ -241,7 +241,7 @@ pub struct Net {
     pub(crate) metrics: Arc<NetDeviceMetrics>,
 
     tx_buffer: IoVecBuffer,
-    rx_buffer: RxBuffers,
+    pub(crate) rx_buffer: RxBuffers,
 }
 
 impl Net {
@@ -436,7 +436,7 @@ impl Net {
     }
 
     /// Parse available RX `DescriptorChains` from the queue
-    fn parse_rx_descriptors(&mut self) {
+    pub fn parse_rx_descriptors(&mut self) {
         // This is safe since we checked in the event handler that the device is activated.
         let mem = self.device_state.mem().unwrap();
         let queue = &mut self.queues[RX_INDEX];

src/vmm/src/devices/virtio/net/persist.rs

@@ -9,9 +9,10 @@ use std::sync::{Arc, Mutex};
 
 use serde::{Deserialize, Serialize};
 
-use super::device::Net;
-use super::{TapError, NET_NUM_QUEUES};
+use super::device::{Net, RxBuffers};
+use super::{TapError, NET_NUM_QUEUES, RX_INDEX};
 use crate::devices::virtio::device::DeviceState;
+use crate::devices::virtio::iovec::ParsedDescriptorChain;
 use crate::devices::virtio::persist::{PersistError as VirtioStateError, VirtioDeviceState};
 use crate::devices::virtio::queue::FIRECRACKER_MAX_QUEUE_SIZE;
 use crate::devices::virtio::TYPE_NET;
@@ -31,6 +32,23 @@ pub struct NetConfigSpaceState {
     guest_mac: Option<MacAddr>,
 }
 
+/// Information about the parsed RX buffers
+#[derive(Debug, Default, Clone, Serialize, Deserialize)]
+pub struct RxBufferState {
+    // Number of iovecs we have parsed from the guest
+    parsed_descriptor_chains_nr: u16,
+    deferred_descriptor: Option<ParsedDescriptorChain>,
+}
+
+impl RxBufferState {
+    fn from_rx_buffers(rx_buffer: &RxBuffers) -> Self {
+        RxBufferState {
+            parsed_descriptor_chains_nr: rx_buffer.parsed_descriptors.len().try_into().unwrap(),
+            deferred_descriptor: rx_buffer.deferred_descriptor.clone(),
+        }
+    }
+}
+
 /// Information about the network device that are saved
 /// at snapshot.
 #[derive(Debug, Clone, Serialize, Deserialize)]
@@ -43,6 +61,7 @@ pub struct NetState {
     pub mmds_ns: Option<MmdsNetworkStackState>,
     config_space: NetConfigSpaceState,
     virtio_state: VirtioDeviceState,
+    rx_buffers_state: RxBufferState,
 }
 
 /// Auxiliary structure for creating a device when resuming from a snapshot.
@@ -85,6 +104,7 @@ impl Persist<'_> for Net {
                 guest_mac: self.guest_mac,
             },
             virtio_state: VirtioDeviceState::from_device(self),
+            rx_buffers_state: RxBufferState::from_rx_buffers(&self.rx_buffer),
         }
     }
 
@@ -137,6 +157,14 @@ impl Persist<'_> for Net {
                 .map_err(NetPersistError::TapSetOffload)?;
 
             net.device_state = DeviceState::Activated(constructor_args.mem);
+
+            // Recreate `Net::rx_buffer`. We do it by re-parsing the RX queue. We're temporarily
+            // rolling back `next_avail` in the RX queue and call `parse_rx_descriptors`.
+            net.queues[RX_INDEX].next_avail -= state.rx_buffers_state.parsed_descriptor_chains_nr;
+            net.parse_rx_descriptors();
+            net.rx_buffer
+                .deferred_descriptor
+                .clone_from(&state.rx_buffers_state.deferred_descriptor);
         }
 
         Ok(net)