Construct and wire up lapics for VTL 1 (#17)

Make the lapics for SNP and Hv backings into an array of 2 instead of
just the 1 per-vp, and wire up the VTLs getting passed around. Next up
is likely taking the result of polling these two apics and using it to
determine which VTL to run in our next run_vp call.

Author: smalis-msft

openhcl/virt_mshv_vtl/src/processor/mod.rs

@@ -207,9 +207,10 @@ mod private {
             stop: &mut StopVp<'_>,
         ) -> impl Future<Output = Result<(), VpHaltReason<UhRunVpError>>>;
 
-        /// Returns true if the VP is ready to run, false if it is halted.
+        /// Returns true if the VP is ready to run the given VTL, false if it is halted.
         fn poll_apic(
             this: &mut UhProcessor<'_, Self>,
+            vtl: Vtl,
             scan_irr: bool,
         ) -> Result<bool, UhRunVpError>;
 
@@ -616,7 +617,7 @@ impl<'p, T: Backing> Processor for UhProcessor<'p, T> {
             Ordering::Relaxed,
         );
 
-        let mut scan_irr = true;
+        let mut first_scan_irr = true;
 
         loop {
             // Process VP activity and wait for the VP to be ready.
@@ -639,22 +640,28 @@ impl<'p, T: Backing> Processor for UhProcessor<'p, T> {
                 }
 
                 // Process wakes.
-                if self.inner.wake_reasons.load(Ordering::Relaxed) != 0 {
-                    scan_irr = self.handle_wake().map_err(VpHaltReason::Hypervisor)?;
+                let scan_irr = if self.inner.wake_reasons.load(Ordering::Relaxed) != 0 {
+                    self.handle_wake().map_err(VpHaltReason::Hypervisor)?
+                } else {
+                    [false, false].into()
+                };
+
+                if self.untrusted_synic.is_some() {
+                    self.update_synic(Vtl::Vtl0, true);
                 }
 
+                // TODO CVM GUEST VSM: Split ready into two to track per-vtl
+                let mut ready = false;
                 for vtl in [Vtl::Vtl1, Vtl::Vtl0] {
                     // Process interrupts.
                     if self.hv(vtl).is_some() {
                         self.update_synic(vtl, false);
                     }
-                }
-                if self.untrusted_synic.is_some() {
-                    self.update_synic(Vtl::Vtl0, true);
-                }
 
-                let ready = T::poll_apic(self, scan_irr).map_err(VpHaltReason::Hypervisor)?;
-                scan_irr = false;
+                    ready |= T::poll_apic(self, vtl, scan_irr[vtl] || first_scan_irr)
+                        .map_err(VpHaltReason::Hypervisor)?;
+                }
+                first_scan_irr = false;
 
                 // Arm the timer.
                 if let Some(timeout) = self.vmtime.get_timeout() {
@@ -696,8 +703,10 @@ impl<'p, T: Backing> Processor for UhProcessor<'p, T> {
     fn flush_async_requests(&mut self) -> Result<(), Self::RunVpError> {
         if self.inner.wake_reasons.load(Ordering::Relaxed) != 0 {
             let scan_irr = self.handle_wake()?;
-            if scan_irr {
-                T::poll_apic(self, true)?;
+            for vtl in [Vtl::Vtl1, Vtl::Vtl0] {
+                if scan_irr[vtl] {
+                    T::poll_apic(self, vtl, true)?;
+                }
             }
         }
         Ok(())
@@ -778,7 +787,7 @@ impl<'a, T: Backing> UhProcessor<'a, T> {
     }
 
     /// Returns true if the interrupt controller has work to do.
-    fn handle_wake(&mut self) -> Result<bool, UhRunVpError> {
+    fn handle_wake(&mut self) -> Result<VtlArray<bool, 2>, UhRunVpError> {
         let wake_reasons_raw = self.inner.wake_reasons.swap(0, Ordering::SeqCst);
         let wake_reasons_vtl: [WakeReason; 2] = zerocopy::transmute!(wake_reasons_raw);
         for (vtl, wake_reasons) in [
@@ -843,7 +852,6 @@ impl<'a, T: Backing> UhProcessor<'a, T> {
                         // Should not have already initialized the hv emulator for this vtl
                         assert!(self.hv(vtl).is_none());
 
-                        // TODO GUEST_VSM construct VTL 1 lapics
                         self.hv[vtl] = Some(
                             self.partition
                                 .hv
@@ -864,7 +872,7 @@ impl<'a, T: Backing> UhProcessor<'a, T> {
             }
         }
 
-        Ok(wake_reasons_vtl[0].intcon())
+        Ok(wake_reasons_vtl.map(|w| w.intcon()).into())
     }
 
     fn request_sint_notifications(&mut self, vtl: Vtl, sints: u16) {

openhcl/virt_mshv_vtl/src/processor/mshv/apic.rs

@@ -35,18 +35,21 @@ use x86defs::RFlags;
 #[derive(Inspect)]
 pub(super) struct UhApicState {
     lapic: LocalApic,
+    #[inspect(debug)]
+    vtl: Vtl,
     pub(super) halted: bool,
     pub(super) startup_suspend: bool,
     nmi_pending: bool,
 }
 
 impl UhApicState {
-    pub fn new(lapic: LocalApic, vp_info: &VpInfo) -> Self {
+    pub fn new(lapic: LocalApic, vtl: Vtl, vp_info: &VpInfo) -> Self {
         Self {
             lapic,
+            vtl,
             halted: false,
             nmi_pending: false,
-            startup_suspend: !vp_info.is_bsp(),
+            startup_suspend: vtl == Vtl::Vtl0 && !vp_info.is_bsp(),
         }
     }
 
@@ -69,6 +72,7 @@ impl UhApicState {
                 runner,
                 dev,
                 vmtime,
+                vtl: self.vtl,
             })
             .mmio_write(address, data);
     }
@@ -88,6 +92,7 @@ impl UhApicState {
                 runner,
                 dev,
                 vmtime,
+                vtl: self.vtl,
             })
             .mmio_read(address, data);
     }
@@ -107,6 +112,7 @@ impl UhApicState {
                 runner,
                 dev,
                 vmtime,
+                vtl: self.vtl,
             })
             .msr_write(msr, value)
     }
@@ -125,6 +131,7 @@ impl UhApicState {
                 runner,
                 dev,
                 vmtime,
+                vtl: self.vtl,
             })
             .msr_read(msr)
     }
@@ -254,10 +261,12 @@ impl UhApicState {
 
 impl UhProcessor<'_, HypervisorBackedX86> {
     /// Returns true if the VP is ready to run, false if it is halted.
-    pub(super) fn poll_apic(&mut self, scan_irr: bool) -> Result<bool, UhRunVpError> {
-        let Some(lapic) = self.backing.lapic.as_mut() else {
+    pub(super) fn poll_apic(&mut self, vtl: Vtl, scan_irr: bool) -> Result<bool, UhRunVpError> {
+        let Some(lapics) = self.backing.lapics.as_mut() else {
             return Ok(true);
         };
+
+        let lapic = &mut lapics[vtl];
         let ApicWork {
             init,
             extint,
@@ -286,33 +295,31 @@ impl UhProcessor<'_, HypervisorBackedX86> {
             todo!();
         }
 
+        // TODO WHP GUEST VSM: An INIT/SIPI targeted at a VP with more than one guest VTL enabled is ignored.
         if init {
-            self.handle_init()?;
+            self.handle_init(vtl)?;
         }
 
         if let Some(vector) = sipi {
-            self.handle_sipi(vector)?;
+            self.handle_sipi(vtl, vector)?;
         }
 
-        let lapic = self.backing.lapic.as_ref().unwrap();
+        let lapic = &self.backing.lapics.as_ref().unwrap()[vtl];
         if lapic.halted || lapic.startup_suspend {
             return Ok(false);
         }
 
         Ok(true)
     }
 
-    fn handle_init(&mut self) -> Result<(), UhRunVpError> {
+    fn handle_init(&mut self, vtl: Vtl) -> Result<(), UhRunVpError> {
         let vp_info = self.inner.vp_info;
-        {
-            let mut access = self.access_state(Vtl::Vtl0);
-            virt::x86::vp::x86_init(&mut access, &vp_info).map_err(UhRunVpError::State)?;
-        }
-        Ok(())
+        let mut access = self.access_state(vtl);
+        virt::x86::vp::x86_init(&mut access, &vp_info).map_err(UhRunVpError::State)
     }
 
-    fn handle_sipi(&mut self, vector: u8) -> Result<(), UhRunVpError> {
-        let lapic = self.backing.lapic.as_mut().unwrap();
+    fn handle_sipi(&mut self, vtl: Vtl, vector: u8) -> Result<(), UhRunVpError> {
+        let lapic = &mut self.backing.lapics.as_mut().unwrap()[vtl];
         if lapic.startup_suspend {
             let address = (vector as u64) << 12;
             let cs: hvdef::HvX64SegmentRegister = hvdef::HvX64SegmentRegister {
@@ -339,6 +346,7 @@ struct UhApicClient<'a, 'b, T> {
     runner: &'a mut ProcessorRunner<'b, MshvX64>,
     dev: &'a T,
     vmtime: &'a VmTimeAccess,
+    vtl: Vtl,
 }
 
 impl<T: CpuIo> ApicClient for UhApicClient<'_, '_, T> {
@@ -365,10 +373,11 @@ impl<T: CpuIo> ApicClient for UhApicClient<'_, '_, T> {
         self.partition
             .vp(vp_index)
             .unwrap()
-            .wake(Vtl::Vtl0, WakeReason::INTCON);
+            .wake(self.vtl, WakeReason::INTCON);
     }
 
     fn eoi(&mut self, vector: u8) {
+        debug_assert_eq!(self.vtl, Vtl::Vtl0);
         self.dev.handle_eoi(vector.into())
     }
 

openhcl/virt_mshv_vtl/src/processor/mshv/arm64.rs

@@ -194,11 +194,12 @@ impl BackingPrivate for HypervisorBackedArm64 {
         Ok(())
     }
 
-    fn poll_apic(this: &mut UhProcessor<'_, Self>, scan_irr: bool) -> Result<bool, UhRunVpError> {
-        {
-            let _ = (this, scan_irr);
-            Ok(true)
-        }
+    fn poll_apic(
+        _this: &mut UhProcessor<'_, Self>,
+        _vtl: Vtl,
+        _scan_irr: bool,
+    ) -> Result<bool, UhRunVpError> {
+        Ok(true)
     }
 
     fn request_extint_readiness(this: &mut UhProcessor<'_, Self>) {

openhcl/virt_mshv_vtl/src/processor/mshv/x64.rs

@@ -60,6 +60,7 @@ use virt_support_x86emu::emulate::EmuCheckVtlAccessError;
 use virt_support_x86emu::emulate::EmuTranslateError;
 use virt_support_x86emu::emulate::EmuTranslateResult;
 use virt_support_x86emu::emulate::EmulatorSupport;
+use vtl_array::VtlArray;
 use vtl_array::VtlSet;
 use x86defs::xsave::Fxsave;
 use x86defs::xsave::XsaveHeader;
@@ -74,7 +75,11 @@ use zerocopy::FromZeroes;
 #[derive(InspectMut)]
 pub struct HypervisorBackedX86 {
     /// Underhill APIC state
-    pub(super) lapic: Option<apic::UhApicState>,
+    pub(super) lapics: Option<VtlArray<apic::UhApicState, 2>>,
+    // TODO WHP GUEST VSM: To be completely correct here, when emulating the APICs
+    // we would need two sets of deliverability notifications too. However currently
+    // we don't support VTL 1 on WHP, and on the hypervisor we don't emulate the APIC,
+    // so this can wait.
     #[inspect(with = "|x| inspect::AsHex(u64::from(*x))")]
     deliverability_notifications: HvDeliverabilityNotificationsRegister,
     /// Next set of deliverability notifications. See register definition for details.
@@ -124,21 +129,27 @@ impl BackingPrivate for HypervisorBackedX86 {
             reserved: [0; 384],
         };
 
-        let lapic = params.partition.lapic.as_ref().map(|arr| {
-            let lapic_set = &arr[Vtl::Vtl0];
+        let lapics = params.partition.lapic.as_ref().map(|arr| {
             // Initialize APIC base to match the current VM state.
             let apic_base = params
                 .runner
                 .get_vp_register(HvX64RegisterName::ApicBase)
                 .unwrap()
                 .as_u64();
-            let mut lapic = lapic_set.add_apic(params.vp_info);
-            lapic.set_apic_base(apic_base).unwrap();
-            apic::UhApicState::new(lapic, &params.vp_info.base)
+            let mut lapic0 = arr[Vtl::Vtl0].add_apic(params.vp_info);
+            lapic0.set_apic_base(apic_base).unwrap();
+            let mut lapic1 = arr[Vtl::Vtl1].add_apic(params.vp_info);
+            lapic1.set_apic_base(apic_base).unwrap();
+
+            [
+                apic::UhApicState::new(lapic0, Vtl::Vtl0, &params.vp_info.base),
+                apic::UhApicState::new(lapic1, Vtl::Vtl1, &params.vp_info.base),
+            ]
+            .into()
         });
 
         Ok(Self {
-            lapic,
+            lapics,
             deliverability_notifications: Default::default(),
             next_deliverability_notifications: Default::default(),
             stats: Default::default(),
@@ -275,8 +286,12 @@ impl BackingPrivate for HypervisorBackedX86 {
         Ok(())
     }
 
-    fn poll_apic(this: &mut UhProcessor<'_, Self>, scan_irr: bool) -> Result<bool, UhRunVpError> {
-        this.poll_apic(scan_irr)
+    fn poll_apic(
+        this: &mut UhProcessor<'_, Self>,
+        vtl: Vtl,
+        scan_irr: bool,
+    ) -> Result<bool, UhRunVpError> {
+        this.poll_apic(vtl, scan_irr)
     }
 
     fn request_extint_readiness(this: &mut UhProcessor<'_, Self>) {
@@ -591,14 +606,21 @@ impl UhProcessor<'_, HypervisorBackedX86> {
             hvdef::HvX64MsrInterceptMessage::ref_from_prefix(self.runner.exit_message().payload())
                 .unwrap();
         let rip = next_rip(&message.header);
+        let last_vtl = self.last_vtl();
 
         tracing::trace!(msg = %format_args!("{:x?}", message), "msr");
 
         let msr = message.msr_number;
         match message.header.intercept_access_type {
             HvInterceptAccessType::READ => {
-                let r = if let Some(lapic) = &mut self.backing.lapic {
-                    lapic.msr_read(self.partition, &mut self.runner, &self.vmtime, dev, msr)
+                let r = if let Some(lapics) = &mut self.backing.lapics {
+                    lapics[last_vtl].msr_read(
+                        self.partition,
+                        &mut self.runner,
+                        &self.vmtime,
+                        dev,
+                        msr,
+                    )
                 } else {
                     Err(MsrError::Unknown)
                 };
@@ -622,8 +644,8 @@ impl UhProcessor<'_, HypervisorBackedX86> {
             }
             HvInterceptAccessType::WRITE => {
                 let value = (message.rax & 0xffff_ffff) | (message.rdx << 32);
-                let r = if let Some(lapic) = &mut self.backing.lapic {
-                    lapic.msr_write(
+                let r = if let Some(lapic) = &mut self.backing.lapics {
+                    lapic[last_vtl].msr_write(
                         self.partition,
                         &mut self.runner,
                         &self.vmtime,
@@ -687,7 +709,8 @@ impl UhProcessor<'_, HypervisorBackedX86> {
     }
 
     fn handle_halt(&mut self) -> Result<(), VpHaltReason<UhRunVpError>> {
-        self.backing.lapic.as_mut().unwrap().halt();
+        let last_vtl = self.last_vtl();
+        self.backing.lapics.as_mut().unwrap()[last_vtl].halt();
         Ok(())
     }
 
@@ -1116,15 +1139,17 @@ impl<T: CpuIo> EmulatorSupport for UhEmulationState<'_, '_, T, HypervisorBackedX
     }
 
     fn lapic_base_address(&self) -> Option<u64> {
+        let last_vtl = self.vp.last_vtl();
         self.vp
             .backing
-            .lapic
+            .lapics
             .as_ref()
-            .and_then(|lapic| lapic.base_address())
+            .and_then(|lapic| lapic[last_vtl].base_address())
     }
 
     fn lapic_read(&mut self, address: u64, data: &mut [u8]) {
-        self.vp.backing.lapic.as_mut().unwrap().mmio_read(
+        let last_vtl = self.vp.last_vtl();
+        self.vp.backing.lapics.as_mut().unwrap()[last_vtl].mmio_read(
             self.vp.partition,
             &mut self.vp.runner,
             &self.vp.vmtime,
@@ -1135,7 +1160,8 @@ impl<T: CpuIo> EmulatorSupport for UhEmulationState<'_, '_, T, HypervisorBackedX
     }
 
     fn lapic_write(&mut self, address: u64, data: &[u8]) {
-        self.vp.backing.lapic.as_mut().unwrap().mmio_write(
+        let last_vtl = self.vp.last_vtl();
+        self.vp.backing.lapics.as_mut().unwrap()[last_vtl].mmio_write(
             self.vp.partition,
             &mut self.vp.runner,
             &self.vp.vmtime,