Skip to content

Commit cf09bac

Browse files
committed
gdb: add support for debugging hyperv guests running on windows
Signed-off-by: Doru Blânzeanu <dblnz@pm.me>
1 parent 51e923e commit cf09bac

File tree

10 files changed

+861
-24
lines changed

10 files changed

+861
-24
lines changed

docs/how-to-debug-a-hyperlight-guest.md

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,12 +1,12 @@
1-
# How to debug a Hyperlight guest using gdb on Linux
1+
# How to debug a Hyperlight guest using gdb or lldb
22

3-
Hyperlight supports gdb debugging of a **KVM** or **MSHV** guest running inside a Hyperlight sandbox on Linux.
3+
Hyperlight supports gdb debugging of a guest running inside a Hyperlight sandbox on Linux or Windows.
44
When Hyperlight is compiled with the `gdb` feature enabled, a Hyperlight sandbox can be configured
55
to start listening for a gdb connection.
66

77
## Supported features
88

9-
The Hyperlight `gdb` feature enables **KVM** and **MSHV** guest debugging to:
9+
The Hyperlight `gdb` feature enables guest debugging to:
1010
- stop at an entry point breakpoint which is automatically set by Hyperlight
1111
- add and remove HW breakpoints (maximum 4 set breakpoints at a time)
1212
- add and remove SW breakpoints
@@ -19,7 +19,7 @@ The Hyperlight `gdb` feature enables **KVM** and **MSHV** guest debugging to:
1919
## Expected behavior
2020

2121
Below is a list describing some cases of expected behavior from a gdb debug
22-
session of a guest binary running inside a Hyperlight sandbox on Linux.
22+
session of a guest binary running inside a Hyperlight sandbox.
2323

2424
- when the `gdb` feature is enabled and a SandboxConfiguration is provided a
2525
debug port, the created sandbox will wait for a gdb client to connect on the

src/hyperlight_host/Cargo.toml

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,8 @@ bench = false # see https://bheisler.github.io/criterion.rs/book/faq.html#cargo-
2121
workspace = true
2222

2323
[dependencies]
24+
gdbstub = { version = "0.7.6", optional = true }
25+
gdbstub_arch = { version = "0.3.2", optional = true }
2426
goblin = { version = "0.10" }
2527
rand = { version = "0.9" }
2628
cfg-if = { version = "1.0.1" }
@@ -67,8 +69,6 @@ windows-version = "0.1"
6769
lazy_static = "1.4.0"
6870

6971
[target.'cfg(unix)'.dependencies]
70-
gdbstub = { version = "0.7.6", optional = true }
71-
gdbstub_arch = { version = "0.3.2", optional = true }
7272
seccompiler = { version = "0.5.0", optional = true }
7373
kvm-bindings = { version = "0.12", features = ["fam-wrappers"], optional = true }
7474
kvm-ioctls = { version = "0.22", optional = true }

src/hyperlight_host/build.rs

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -90,7 +90,7 @@ fn main() -> Result<()> {
9090
// Essentially the kvm and mshv features are ignored on windows as long as you use #[cfg(kvm)] and not #[cfg(feature = "kvm")].
9191
// You should never use #[cfg(feature = "kvm")] or #[cfg(feature = "mshv")] in the codebase.
9292
cfg_aliases::cfg_aliases! {
93-
gdb: { all(feature = "gdb", debug_assertions, any(feature = "kvm", feature = "mshv2", feature = "mshv3"), target_os = "linux") },
93+
gdb: { all(feature = "gdb", debug_assertions, any(feature = "kvm", feature = "mshv2", feature = "mshv3")) },
9494
kvm: { all(feature = "kvm", target_os = "linux") },
9595
mshv: { all(any(feature = "mshv2", feature = "mshv3"), target_os = "linux") },
9696
crashdump: { all(feature = "crashdump") },

src/hyperlight_host/src/hypervisor/gdb/event_loop.rs

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -22,9 +22,17 @@ use gdbstub::stub::{
2222

2323
use super::x86_64_target::HyperlightSandboxTarget;
2424
use super::{DebugResponse, GdbTargetError, VcpuStopReason};
25+
26+
// Signals are defined differently on Windows and Linux, so we use conditional compilation
27+
#[cfg(target_os = "linux")]
2528
mod signals {
2629
pub use libc::{SIGINT, SIGSEGV};
2730
}
31+
#[cfg(windows)]
32+
mod signals {
33+
pub const SIGINT: i8 = 2;
34+
pub const SIGSEGV: i8 = 11;
35+
}
2836

2937
struct GdbBlockingEventLoop;
3038

Lines changed: 255 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,255 @@
1+
/*
2+
Copyright 2024 The Hyperlight Authors.
3+
4+
Licensed under the Apache License, Version 2.0 (the "License");
5+
you may not use this file except in compliance with the License.
6+
You may obtain a copy of the License at
7+
8+
http://www.apache.org/licenses/LICENSE-2.0
9+
10+
Unless required by applicable law or agreed to in writing, software
11+
distributed under the License is distributed on an "AS IS" BASIS,
12+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
See the License for the specific language governing permissions and
14+
limitations under the License.
15+
*/
16+
17+
use std::collections::HashMap;
18+
19+
use windows::Win32::System::Hypervisor::WHV_VP_EXCEPTION_CONTEXT;
20+
21+
use super::arch::{MAX_NO_OF_HW_BP, vcpu_stop_reason};
22+
use super::{GuestDebug, SW_BP_SIZE, VcpuStopReason, X86_64Regs};
23+
use crate::hypervisor::windows_hypervisor_platform::VMProcessor;
24+
use crate::hypervisor::wrappers::{WHvDebugRegisters, WHvGeneralRegisters};
25+
use crate::{HyperlightError, Result, new_error};
26+
27+
/// KVM Debug struct
28+
/// This struct is used to abstract the internal details of the kvm
29+
/// guest debugging settings
30+
#[derive(Default)]
31+
pub(crate) struct HypervDebug {
32+
/// vCPU stepping state
33+
single_step: bool,
34+
35+
/// Array of addresses for HW breakpoints
36+
hw_breakpoints: Vec<u64>,
37+
/// Saves the bytes modified to enable SW breakpoints
38+
sw_breakpoints: HashMap<u64, [u8; SW_BP_SIZE]>,
39+
40+
/// Debug registers
41+
dbg_cfg: WHvDebugRegisters,
42+
}
43+
44+
impl HypervDebug {
45+
pub(crate) fn new() -> Self {
46+
Self {
47+
single_step: false,
48+
hw_breakpoints: vec![],
49+
sw_breakpoints: HashMap::new(),
50+
dbg_cfg: WHvDebugRegisters::default(),
51+
}
52+
}
53+
54+
/// Returns the instruction pointer from the stopped vCPU
55+
fn get_instruction_pointer(&self, vcpu_fd: &VMProcessor) -> Result<u64> {
56+
let regs = vcpu_fd
57+
.get_regs()
58+
.map_err(|e| new_error!("Could not retrieve registers from vCPU: {:?}", e))?;
59+
60+
Ok(regs.rip)
61+
}
62+
63+
/// This method sets the kvm debugreg fields to enable breakpoints at
64+
/// specific addresses
65+
///
66+
/// The first 4 debug registers are used to set the addresses
67+
/// The 4th and 5th debug registers are obsolete and not used
68+
/// The 7th debug register is used to enable the breakpoints
69+
/// For more information see: DEBUG REGISTERS chapter in the architecture
70+
/// manual
71+
fn set_debug_config(&mut self, vcpu_fd: &VMProcessor, step: bool) -> Result<()> {
72+
let addrs = &self.hw_breakpoints;
73+
74+
let mut dbg_cfg = WHvDebugRegisters::default();
75+
76+
for (k, addr) in addrs.iter().enumerate() {
77+
match k {
78+
0 => {
79+
dbg_cfg.dr0 = *addr;
80+
}
81+
1 => {
82+
dbg_cfg.dr1 = *addr;
83+
}
84+
2 => {
85+
dbg_cfg.dr2 = *addr;
86+
}
87+
3 => {
88+
dbg_cfg.dr3 = *addr;
89+
}
90+
_ => {
91+
Err(new_error!("Tried to set more than 4 HW breakpoints"))?;
92+
}
93+
}
94+
dbg_cfg.dr7 |= 1 << (k * 2);
95+
}
96+
97+
self.dbg_cfg = dbg_cfg;
98+
99+
vcpu_fd
100+
.set_debug_regs(&self.dbg_cfg)
101+
.map_err(|e| new_error!("Could not set guest debug: {:?}", e))?;
102+
103+
self.single_step = step;
104+
105+
let mut regs = vcpu_fd
106+
.get_regs()
107+
.map_err(|e| new_error!("Could not get registers: {:?}", e))?;
108+
109+
// Set TF Flag to enable Traps
110+
if self.single_step {
111+
regs.rflags |= 1 << 8; // Set the TF flag
112+
} else {
113+
regs.rflags &= !(1 << 8); // Clear the TF flag
114+
}
115+
116+
vcpu_fd
117+
.set_general_purpose_registers(&regs)
118+
.map_err(|e| new_error!("Could not set guest registers: {:?}", e))?;
119+
120+
Ok(())
121+
}
122+
123+
/// Get the reason the vCPU has stopped
124+
pub(crate) fn get_stop_reason(
125+
&mut self,
126+
vcpu_fd: &VMProcessor,
127+
exception: WHV_VP_EXCEPTION_CONTEXT,
128+
entrypoint: u64,
129+
) -> Result<VcpuStopReason> {
130+
let rip = self.get_instruction_pointer(vcpu_fd)?;
131+
let rip = self.translate_gva(vcpu_fd, rip)?;
132+
133+
let debug_regs = vcpu_fd
134+
.get_debug_regs()
135+
.map_err(|e| new_error!("Could not retrieve registers from vCPU: {:?}", e))?;
136+
137+
// Check if the vCPU stopped because of a hardware breakpoint
138+
let reason = vcpu_stop_reason(
139+
self.single_step,
140+
rip,
141+
debug_regs.dr6,
142+
entrypoint,
143+
exception.ExceptionType as u32,
144+
&self.hw_breakpoints,
145+
&self.sw_breakpoints,
146+
);
147+
148+
if let VcpuStopReason::EntryPointBp = reason {
149+
// In case the hw breakpoint is the entry point, remove it to
150+
// avoid hanging here as gdb does not remove breakpoints it
151+
// has not set.
152+
// Gdb expects the target to be stopped when connected.
153+
self.remove_hw_breakpoint(vcpu_fd, entrypoint)?;
154+
}
155+
156+
Ok(reason)
157+
}
158+
}
159+
160+
impl GuestDebug for HypervDebug {
161+
type Vcpu = VMProcessor;
162+
163+
fn is_hw_breakpoint(&self, addr: &u64) -> bool {
164+
self.hw_breakpoints.contains(addr)
165+
}
166+
fn is_sw_breakpoint(&self, addr: &u64) -> bool {
167+
self.sw_breakpoints.contains_key(addr)
168+
}
169+
fn save_hw_breakpoint(&mut self, addr: &u64) -> bool {
170+
if self.hw_breakpoints.len() >= MAX_NO_OF_HW_BP {
171+
false
172+
} else {
173+
self.hw_breakpoints.push(*addr);
174+
175+
true
176+
}
177+
}
178+
fn save_sw_breakpoint_data(&mut self, addr: u64, data: [u8; 1]) {
179+
_ = self.sw_breakpoints.insert(addr, data);
180+
}
181+
fn delete_hw_breakpoint(&mut self, addr: &u64) {
182+
self.hw_breakpoints.retain(|&a| a != *addr);
183+
}
184+
fn delete_sw_breakpoint_data(&mut self, addr: &u64) -> Option<[u8; 1]> {
185+
self.sw_breakpoints.remove(addr)
186+
}
187+
188+
fn read_regs(&self, vcpu_fd: &Self::Vcpu, regs: &mut X86_64Regs) -> Result<()> {
189+
log::debug!("Read registers");
190+
let vcpu_regs = vcpu_fd
191+
.get_regs()
192+
.map_err(|e| new_error!("Could not read guest registers: {:?}", e))?;
193+
194+
regs.rax = vcpu_regs.rax;
195+
regs.rbx = vcpu_regs.rbx;
196+
regs.rcx = vcpu_regs.rcx;
197+
regs.rdx = vcpu_regs.rdx;
198+
regs.rsi = vcpu_regs.rsi;
199+
regs.rdi = vcpu_regs.rdi;
200+
regs.rbp = vcpu_regs.rbp;
201+
regs.rsp = vcpu_regs.rsp;
202+
regs.r8 = vcpu_regs.r8;
203+
regs.r9 = vcpu_regs.r9;
204+
regs.r10 = vcpu_regs.r10;
205+
regs.r11 = vcpu_regs.r11;
206+
regs.r12 = vcpu_regs.r12;
207+
regs.r13 = vcpu_regs.r13;
208+
regs.r14 = vcpu_regs.r14;
209+
regs.r15 = vcpu_regs.r15;
210+
211+
regs.rip = vcpu_regs.rip;
212+
regs.rflags = vcpu_regs.rflags;
213+
214+
Ok(())
215+
}
216+
217+
fn set_single_step(&mut self, vcpu_fd: &Self::Vcpu, enable: bool) -> Result<()> {
218+
self.set_debug_config(vcpu_fd, enable)
219+
}
220+
221+
fn translate_gva(&self, vcpu_fd: &Self::Vcpu, gva: u64) -> Result<u64> {
222+
vcpu_fd
223+
.translate_gva(gva)
224+
.map_err(|_| HyperlightError::TranslateGuestAddress(gva))
225+
}
226+
227+
fn write_regs(&self, vcpu_fd: &Self::Vcpu, regs: &X86_64Regs) -> Result<()> {
228+
log::debug!("Write registers");
229+
let regs = WHvGeneralRegisters {
230+
rax: regs.rax,
231+
rbx: regs.rbx,
232+
rcx: regs.rcx,
233+
rdx: regs.rdx,
234+
rsi: regs.rsi,
235+
rdi: regs.rdi,
236+
rbp: regs.rbp,
237+
rsp: regs.rsp,
238+
r8: regs.r8,
239+
r9: regs.r9,
240+
r10: regs.r10,
241+
r11: regs.r11,
242+
r12: regs.r12,
243+
r13: regs.r13,
244+
r14: regs.r14,
245+
r15: regs.r15,
246+
247+
rip: regs.rip,
248+
rflags: regs.rflags,
249+
};
250+
251+
vcpu_fd
252+
.set_general_purpose_registers(&regs)
253+
.map_err(|e| new_error!("Could not write guest registers: {:?}", e))
254+
}
255+
}

src/hyperlight_host/src/hypervisor/gdb/mod.rs

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,8 @@ limitations under the License.
1616

1717
mod arch;
1818
mod event_loop;
19+
#[cfg(target_os = "windows")]
20+
mod hyperv_debug;
1921
#[cfg(kvm)]
2022
mod kvm_debug;
2123
#[cfg(mshv)]
@@ -34,6 +36,8 @@ use gdbstub::conn::ConnectionExt;
3436
use gdbstub::stub::GdbStub;
3537
use gdbstub::target::TargetError;
3638
use hyperlight_common::mem::PAGE_SIZE;
39+
#[cfg(target_os = "windows")]
40+
pub(crate) use hyperv_debug::HypervDebug;
3741
#[cfg(kvm)]
3842
pub(crate) use kvm_debug::KvmDebug;
3943
#[cfg(mshv)]

0 commit comments

Comments
 (0)