From e2832da01b048ea297d99cf63da67ccfe88fa2f2 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Doru=20Bl=C3=A2nzeanu?= <dblnz@pm.me>
Date: Tue, 4 Feb 2025 15:16:57 +0200
Subject: [PATCH 01/18] add gdb cargo feature
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Signed-off-by: Doru Blânzeanu <dblnz@pm.me>
---
 src/hyperlight_host/Cargo.toml | 2 ++
 src/hyperlight_host/build.rs   | 1 +
 2 files changed, 3 insertions(+)

diff --git a/src/hyperlight_host/Cargo.toml b/src/hyperlight_host/Cargo.toml
index 02a964fa..84acc004 100644
--- a/src/hyperlight_host/Cargo.toml
+++ b/src/hyperlight_host/Cargo.toml
@@ -128,6 +128,8 @@ kvm = ["dep:kvm-bindings", "dep:kvm-ioctls"]
 mshv2 = ["dep:mshv-bindings2", "dep:mshv-ioctls2"]
 mshv3 = ["dep:mshv-bindings3", "dep:mshv-ioctls3"]
 inprocess = []
+# This enables compilation of gdb stub for easy debug in the guest
+gdb = []
 
 [[bench]]
 name = "benchmarks"
diff --git a/src/hyperlight_host/build.rs b/src/hyperlight_host/build.rs
index 7600f647..a8370725 100644
--- a/src/hyperlight_host/build.rs
+++ b/src/hyperlight_host/build.rs
@@ -89,6 +89,7 @@ fn main() -> Result<()> {
     // Essentially the kvm and mshv features are ignored on windows as long as you use #[cfg(kvm)] and not #[cfg(feature = "kvm")].
     // You should never use #[cfg(feature = "kvm")] or #[cfg(feature = "mshv")] in the codebase.
     cfg_aliases::cfg_aliases! {
+        gdb: { all(feature = "gdb", debug_assertions, feature = "kvm", target_os = "linux") },
         kvm: { all(feature = "kvm", target_os = "linux") },
         mshv: { all(any(feature = "mshv2", feature = "mshv3"), target_os = "linux") },
         // inprocess feature is aliased with debug_assertions to make it only available in debug-builds.

From 77972adcef772f74695dfaaff10714982152cecc Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Doru=20Bl=C3=A2nzeanu?= <dblnz@pm.me>
Date: Tue, 4 Feb 2025 15:33:47 +0200
Subject: [PATCH 02/18] disable timeouts for messaging between host and
 hypervisor handler when gdb feature is on
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Signed-off-by: Doru Blânzeanu <dblnz@pm.me>
---
 .../src/hypervisor/hypervisor_handler.rs           | 14 +++++++++++---
 1 file changed, 11 insertions(+), 3 deletions(-)

diff --git a/src/hyperlight_host/src/hypervisor/hypervisor_handler.rs b/src/hyperlight_host/src/hypervisor/hypervisor_handler.rs
index a221570a..8e69f3fb 100644
--- a/src/hyperlight_host/src/hypervisor/hypervisor_handler.rs
+++ b/src/hyperlight_host/src/hypervisor/hypervisor_handler.rs
@@ -597,11 +597,19 @@ impl HypervisorHandler {
     /// and still have to receive after sorting that out without sending
     /// an extra message.
     pub(crate) fn try_receive_handler_msg(&self) -> Result<()> {
-        match self
+        // When gdb debugging is enabled, we don't want to timeout on receiving messages
+        // from the handler thread, as the thread may be paused by gdb.
+        // In this case, we will wait indefinitely for a message from the handler thread.
+        // Note: This applies to all the running sandboxes, not just the one being debugged.
+        #[cfg(gdb)]
+        let response = self.communication_channels.from_handler_rx.recv();
+        #[cfg(not(gdb))]
+        let response = self
             .communication_channels
             .from_handler_rx
-            .recv_timeout(self.execution_variables.get_timeout()?)
-        {
+            .recv_timeout(self.execution_variables.get_timeout()?);
+
+        match response {
             Ok(msg) => match msg {
                 HandlerMsg::Error(e) => Err(e),
                 HandlerMsg::FinishedHypervisorHandlerAction => Ok(()),

From 5b7de0da4d272c6c0a7b2bbf38cbdfba551ddecd Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Doru=20Bl=C3=A2nzeanu?= <dblnz@pm.me>
Date: Tue, 4 Feb 2025 16:34:47 +0200
Subject: [PATCH 03/18] add guest debug port to sandbox configuration
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Signed-off-by: Doru Blânzeanu <dblnz@pm.me>
---
 .../src/hypervisor/hypervisor_handler.rs      |  6 +++
 src/hyperlight_host/src/hypervisor/mod.rs     |  6 ++-
 src/hyperlight_host/src/sandbox/config.rs     | 44 +++++++++++++++++++
 .../src/sandbox/uninitialized.rs              |  8 ++++
 .../src/sandbox/uninitialized_evolve.rs       | 11 ++++-
 5 files changed, 73 insertions(+), 2 deletions(-)

diff --git a/src/hyperlight_host/src/hypervisor/hypervisor_handler.rs b/src/hyperlight_host/src/hypervisor/hypervisor_handler.rs
index 8e69f3fb..d960ff0e 100644
--- a/src/hyperlight_host/src/hypervisor/hypervisor_handler.rs
+++ b/src/hyperlight_host/src/hypervisor/hypervisor_handler.rs
@@ -46,6 +46,8 @@ use crate::mem::mgr::SandboxMemoryManager;
 use crate::mem::ptr::{GuestPtr, RawPtr};
 use crate::mem::ptr_offset::Offset;
 use crate::mem::shared_mem::{GuestSharedMemory, HostSharedMemory, SharedMemory};
+#[cfg(gdb)]
+use crate::sandbox::config::DebugInfo;
 use crate::sandbox::hypervisor::{get_available_hypervisor, HypervisorType};
 #[cfg(feature = "function_call_metrics")]
 use crate::sandbox::metrics::SandboxMetric::GuestFunctionCallDurationMicroseconds;
@@ -232,6 +234,7 @@ impl HypervisorHandler {
     pub(crate) fn start_hypervisor_handler(
         &mut self,
         sandbox_memory_manager: SandboxMemoryManager<GuestSharedMemory>,
+        #[cfg(gdb)] debug_info: Option<DebugInfo>,
     ) -> Result<()> {
         let configuration = self.configuration.clone();
         #[cfg(target_os = "windows")]
@@ -292,6 +295,8 @@ impl HypervisorHandler {
                                     hv = Some(set_up_hypervisor_partition(
                                         execution_variables.shm.try_lock().unwrap().deref_mut().as_mut().unwrap(),
                                         configuration.outb_handler.clone(),
+                                        #[cfg(gdb)]
+                                        &debug_info,
                                     )?);
                                 }
                                 let hv = hv.as_mut().unwrap();
@@ -831,6 +836,7 @@ fn set_up_hypervisor_partition(
     mgr: &mut SandboxMemoryManager<GuestSharedMemory>,
     #[allow(unused_variables)] // parameter only used for in-process mode
     outb_handler: OutBHandlerWrapper,
+    #[cfg(gdb)] _debug_info: &Option<DebugInfo>,
 ) -> Result<Box<dyn Hypervisor>> {
     let mem_size = u64::try_from(mgr.shared_mem.mem_size())?;
     let mut regions = mgr.layout.get_memory_regions(&mgr.shared_mem)?;
diff --git a/src/hyperlight_host/src/hypervisor/mod.rs b/src/hyperlight_host/src/hypervisor/mod.rs
index c34d7eb1..62ec8ca8 100644
--- a/src/hyperlight_host/src/hypervisor/mod.rs
+++ b/src/hyperlight_host/src/hypervisor/mod.rs
@@ -336,7 +336,11 @@ pub(crate) mod tests {
         // whether we can configure the shared memory region, load a binary
         // into it, and run the CPU to completion (e.g., a HLT interrupt)
 
-        hv_handler.start_hypervisor_handler(gshm)?;
+        hv_handler.start_hypervisor_handler(
+            gshm,
+            #[cfg(gdb)]
+            None,
+        )?;
 
         hv_handler.execute_hypervisor_handler_action(HypervisorHandlerAction::Initialise)
     }
diff --git a/src/hyperlight_host/src/sandbox/config.rs b/src/hyperlight_host/src/sandbox/config.rs
index c03528e3..e0bbaddb 100644
--- a/src/hyperlight_host/src/sandbox/config.rs
+++ b/src/hyperlight_host/src/sandbox/config.rs
@@ -21,10 +21,21 @@ use tracing::{instrument, Span};
 
 use crate::mem::exe::ExeInfo;
 
+/// Used for passing debug configuration to a sandbox
+#[cfg(gdb)]
+#[derive(Copy, Clone, Debug, Eq, PartialEq)]
+pub struct DebugInfo {
+    /// Guest debug port
+    pub port: u16,
+}
+
 /// The complete set of configuration needed to create a Sandbox
 #[derive(Copy, Clone, Debug, Eq, PartialEq)]
 #[repr(C)]
 pub struct SandboxConfiguration {
+    /// Guest gdb debug port
+    #[cfg(gdb)]
+    guest_debug_info: Option<DebugInfo>,
     /// The maximum size of the guest error buffer.
     guest_error_buffer_size: usize,
     /// The size of the memory buffer that is made available for Guest Function
@@ -153,6 +164,7 @@ impl SandboxConfiguration {
         max_initialization_time: Option<Duration>,
         max_wait_for_cancellation: Option<Duration>,
         guest_panic_context_buffer_size: usize,
+        #[cfg(gdb)] guest_debug_info: Option<DebugInfo>,
     ) -> Self {
         Self {
             input_data_size: max(input_data_size, Self::MIN_INPUT_SIZE),
@@ -220,6 +232,8 @@ impl SandboxConfiguration {
                 guest_panic_context_buffer_size,
                 Self::MIN_GUEST_PANIC_CONTEXT_BUFFER_SIZE,
             ),
+            #[cfg(gdb)]
+            guest_debug_info,
         }
     }
 
@@ -346,6 +360,13 @@ impl SandboxConfiguration {
         );
     }
 
+    /// Sets the configuration for the guest debug
+    #[cfg(gdb)]
+    #[instrument(skip_all, parent = Span::current(), level= "Trace")]
+    pub fn set_guest_debug_info(&mut self, debug_info: DebugInfo) {
+        self.guest_debug_info = Some(debug_info);
+    }
+
     #[instrument(skip_all, parent = Span::current(), level= "Trace")]
     pub(crate) fn get_guest_error_buffer_size(&self) -> usize {
         self.guest_error_buffer_size
@@ -390,6 +411,12 @@ impl SandboxConfiguration {
         self.max_initialization_time
     }
 
+    #[cfg(gdb)]
+    #[instrument(skip_all, parent = Span::current(), level= "Trace")]
+    pub(crate) fn get_guest_debug_info(&self) -> Option<DebugInfo> {
+        self.guest_debug_info
+    }
+
     #[instrument(skip_all, parent = Span::current(), level= "Trace")]
     fn stack_size_override_opt(&self) -> Option<u64> {
         (self.stack_size_override > 0).then_some(self.stack_size_override)
@@ -438,6 +465,8 @@ impl Default for SandboxConfiguration {
             None,
             None,
             Self::DEFAULT_GUEST_PANIC_CONTEXT_BUFFER_SIZE,
+            #[cfg(gdb)]
+            None,
         )
     }
 }
@@ -480,6 +509,8 @@ mod tests {
                 MAX_WAIT_FOR_CANCELLATION_OVERRIDE as u64,
             )),
             GUEST_PANIC_CONTEXT_BUFFER_SIZE_OVERRIDE,
+            #[cfg(gdb)]
+            None,
         );
         let exe_infos = vec![
             simple_guest_exe_info().unwrap(),
@@ -543,6 +574,8 @@ mod tests {
                 SandboxConfiguration::MIN_MAX_WAIT_FOR_CANCELLATION as u64 - 1,
             )),
             SandboxConfiguration::MIN_GUEST_PANIC_CONTEXT_BUFFER_SIZE - 1,
+            #[cfg(gdb)]
+            None,
         );
         assert_eq!(SandboxConfiguration::MIN_INPUT_SIZE, cfg.input_data_size);
         assert_eq!(SandboxConfiguration::MIN_OUTPUT_SIZE, cfg.output_data_size);
@@ -633,6 +666,8 @@ mod tests {
         use proptest::prelude::*;
 
         use super::SandboxConfiguration;
+        #[cfg(gdb)]
+        use crate::sandbox::config::DebugInfo;
 
         proptest! {
             #[test]
@@ -711,6 +746,15 @@ mod tests {
                 cfg.set_heap_size(size);
                 prop_assert_eq!(size, cfg.heap_size_override);
             }
+
+            #[test]
+            #[cfg(gdb)]
+            fn guest_debug_info(port in 9000..=u16::MAX) {
+                let mut cfg = SandboxConfiguration::default();
+                let debug_info = DebugInfo { port };
+                cfg.set_guest_debug_info(debug_info);
+                prop_assert_eq!(debug_info, *cfg.get_guest_debug_info().as_ref().unwrap());
+            }
         }
     }
 }
diff --git a/src/hyperlight_host/src/sandbox/uninitialized.rs b/src/hyperlight_host/src/sandbox/uninitialized.rs
index 4ca951d5..e3f8e0bd 100644
--- a/src/hyperlight_host/src/sandbox/uninitialized.rs
+++ b/src/hyperlight_host/src/sandbox/uninitialized.rs
@@ -22,6 +22,8 @@ use std::time::Duration;
 
 use tracing::{instrument, Span};
 
+#[cfg(gdb)]
+use super::config::DebugInfo;
 use super::host_funcs::{default_writer_func, HostFuncsWrapper};
 use super::mem_mgr::MemMgrWrapper;
 use super::run_options::SandboxRunOptions;
@@ -52,6 +54,8 @@ pub struct UninitializedSandbox {
     pub(crate) max_initialization_time: Duration,
     pub(crate) max_execution_time: Duration,
     pub(crate) max_wait_for_cancellation: Duration,
+    #[cfg(gdb)]
+    pub(crate) debug_info: Option<DebugInfo>,
 }
 
 impl crate::sandbox_state::sandbox::UninitializedSandbox for UninitializedSandbox {
@@ -161,6 +165,8 @@ impl UninitializedSandbox {
         }
 
         let sandbox_cfg = cfg.unwrap_or_default();
+        #[cfg(gdb)]
+        let debug_info = sandbox_cfg.get_guest_debug_info();
         let mut mem_mgr_wrapper = {
             let mut mgr = UninitializedSandbox::load_guest_binary(
                 sandbox_cfg,
@@ -188,6 +194,8 @@ impl UninitializedSandbox {
             max_wait_for_cancellation: Duration::from_millis(
                 sandbox_cfg.get_max_wait_for_cancellation() as u64,
             ),
+            #[cfg(gdb)]
+            debug_info,
         };
 
         // TODO: These only here to accommodate some writer functions.
diff --git a/src/hyperlight_host/src/sandbox/uninitialized_evolve.rs b/src/hyperlight_host/src/sandbox/uninitialized_evolve.rs
index ccbed836..55252c92 100644
--- a/src/hyperlight_host/src/sandbox/uninitialized_evolve.rs
+++ b/src/hyperlight_host/src/sandbox/uninitialized_evolve.rs
@@ -26,6 +26,8 @@ use crate::hypervisor::hypervisor_handler::{
 use crate::mem::mgr::SandboxMemoryManager;
 use crate::mem::ptr::RawPtr;
 use crate::mem::shared_mem::GuestSharedMemory;
+#[cfg(gdb)]
+use crate::sandbox::config::DebugInfo;
 use crate::sandbox::host_funcs::HostFuncsWrapper;
 use crate::sandbox::mem_access::mem_access_handler_wrapper;
 use crate::sandbox::outb::outb_handler_wrapper;
@@ -66,6 +68,8 @@ where
             u_sbox.max_initialization_time,
             u_sbox.max_execution_time,
             u_sbox.max_wait_for_cancellation,
+            #[cfg(gdb)]
+            u_sbox.debug_info,
         )?;
 
         {
@@ -98,6 +102,7 @@ fn hv_init(
     max_init_time: Duration,
     max_exec_time: Duration,
     max_wait_for_cancellation: Duration,
+    #[cfg(gdb)] debug_info: Option<DebugInfo>,
 ) -> Result<HypervisorHandler> {
     let outb_hdl = outb_handler_wrapper(hshm.clone(), host_funcs);
     let mem_access_hdl = mem_access_handler_wrapper(hshm.clone());
@@ -126,7 +131,11 @@ fn hv_init(
 
     let mut hv_handler = HypervisorHandler::new(hv_handler_config);
 
-    hv_handler.start_hypervisor_handler(gshm)?;
+    hv_handler.start_hypervisor_handler(
+        gshm,
+        #[cfg(gdb)]
+        debug_info,
+    )?;
 
     hv_handler
         .execute_hypervisor_handler_action(HypervisorHandlerAction::Initialise)

From 03c68a15433a1c55e5fbe0af18a09fc9659e352e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Doru=20Bl=C3=A2nzeanu?= <dblnz@pm.me>
Date: Tue, 4 Feb 2025 17:16:57 +0200
Subject: [PATCH 04/18] add gdb debug tread creation method
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Signed-off-by: Doru Blânzeanu <dblnz@pm.me>
---
 Cargo.lock                                    | 32 ++++++++++
 src/hyperlight_host/Cargo.toml                |  6 +-
 src/hyperlight_host/src/hypervisor/gdb/mod.rs | 62 +++++++++++++++++++
 .../src/hypervisor/hypervisor_handler.rs      |  4 +-
 src/hyperlight_host/src/hypervisor/kvm.rs     | 10 +++
 src/hyperlight_host/src/hypervisor/mod.rs     |  4 ++
 6 files changed, 115 insertions(+), 3 deletions(-)
 create mode 100644 src/hyperlight_host/src/hypervisor/gdb/mod.rs

diff --git a/Cargo.lock b/Cargo.lock
index 5c52341e..27c915a4 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -805,6 +805,30 @@ dependencies = [
  "slab",
 ]
 
+[[package]]
+name = "gdbstub"
+version = "0.7.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "31c683a9f13de31432e6097131d5f385898c7f0635c0f392b9d0fa165063c8ac"
+dependencies = [
+ "bitflags 2.8.0",
+ "cfg-if",
+ "log",
+ "managed",
+ "num-traits",
+ "paste",
+]
+
+[[package]]
+name = "gdbstub_arch"
+version = "0.3.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "328a9e9425db13770d0d11de6332a608854266e44c53d12776be7b4aa427e3de"
+dependencies = [
+ "gdbstub",
+ "num-traits",
+]
+
 [[package]]
 name = "generic-array"
 version = "0.14.7"
@@ -1106,6 +1130,8 @@ dependencies = [
  "env_logger",
  "envy",
  "flatbuffers",
+ "gdbstub",
+ "gdbstub_arch",
  "goblin",
  "hyperlight-common",
  "hyperlight-testing",
@@ -1568,6 +1594,12 @@ dependencies = [
  "libc",
 ]
 
+[[package]]
+name = "managed"
+version = "0.8.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0ca88d725a0a943b096803bd34e73a4437208b6077654cc4ecb2947a5f91618d"
+
 [[package]]
 name = "matchers"
 version = "0.1.0"
diff --git a/src/hyperlight_host/Cargo.toml b/src/hyperlight_host/Cargo.toml
index 84acc004..ac1431bd 100644
--- a/src/hyperlight_host/Cargo.toml
+++ b/src/hyperlight_host/Cargo.toml
@@ -71,6 +71,8 @@ sha256 = "1.4.0"
 windows-version = "0.1"
 
 [target.'cfg(unix)'.dependencies]
+gdbstub = { version = "0.7.3", optional = true }
+gdbstub_arch = { version = "0.3.1", optional = true }
 seccompiler = { version = "0.4.0", optional = true }
 kvm-bindings = { version = "0.11", features = ["fam-wrappers"], optional = true }
 kvm-ioctls = { version = "0.20", optional = true }
@@ -128,8 +130,8 @@ kvm = ["dep:kvm-bindings", "dep:kvm-ioctls"]
 mshv2 = ["dep:mshv-bindings2", "dep:mshv-ioctls2"]
 mshv3 = ["dep:mshv-bindings3", "dep:mshv-ioctls3"]
 inprocess = []
-# This enables compilation of gdb stub for easy debug in the guest
-gdb = []
+# This enables easy debug in the guest
+gdb = ["dep:gdbstub", "dep:gdbstub_arch"]
 
 [[bench]]
 name = "benchmarks"
diff --git a/src/hyperlight_host/src/hypervisor/gdb/mod.rs b/src/hyperlight_host/src/hypervisor/gdb/mod.rs
new file mode 100644
index 00000000..340d41e2
--- /dev/null
+++ b/src/hyperlight_host/src/hypervisor/gdb/mod.rs
@@ -0,0 +1,62 @@
+/*
+Copyright 2024 The Hyperlight Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+use std::io::{self, ErrorKind};
+use std::net::TcpListener;
+use std::thread;
+use thiserror::Error;
+#[derive(Debug, Error)]
+pub enum GdbTargetError {
+    #[error("Error encountered while binding to address and port")]
+    CannotBind,
+    #[error("Error encountered while listening for connections")]
+    ListenerError,
+    #[error("Unexpected error encountered")]
+    UnexpectedError,
+}
+
+impl From<io::Error> for GdbTargetError {
+    fn from(err: io::Error) -> Self {
+        match err.kind() {
+            ErrorKind::AddrInUse => Self::CannotBind,
+            ErrorKind::AddrNotAvailable => Self::CannotBind,
+            ErrorKind::ConnectionReset
+            | ErrorKind::ConnectionAborted
+            | ErrorKind::ConnectionRefused => Self::ListenerError,
+            _ => Self::UnexpectedError,
+        }
+    }
+}
+/// Creates a thread that handles gdb protocol
+pub fn create_gdb_thread(
+    port: u16,
+) -> Result<(), GdbTargetError> {
+    let socket = format!("localhost:{}", port);
+
+    log::info!("Listening on {:?}", socket);
+    let listener = TcpListener::bind(socket)?;
+
+    log::info!("Starting GDB thread");
+    let _handle = thread::Builder::new()
+        .name("GDB handler".to_string())
+        .spawn(move || -> Result<(), GdbTargetError> {
+            log::info!("Waiting for GDB connection ... ");
+            let (_, _) = listener.accept()?;
+            Ok(())
+        });
+
+    Ok(())
+}
diff --git a/src/hyperlight_host/src/hypervisor/hypervisor_handler.rs b/src/hyperlight_host/src/hypervisor/hypervisor_handler.rs
index d960ff0e..cd1b8b91 100644
--- a/src/hyperlight_host/src/hypervisor/hypervisor_handler.rs
+++ b/src/hyperlight_host/src/hypervisor/hypervisor_handler.rs
@@ -836,7 +836,7 @@ fn set_up_hypervisor_partition(
     mgr: &mut SandboxMemoryManager<GuestSharedMemory>,
     #[allow(unused_variables)] // parameter only used for in-process mode
     outb_handler: OutBHandlerWrapper,
-    #[cfg(gdb)] _debug_info: &Option<DebugInfo>,
+    #[cfg(gdb)] debug_info: &Option<DebugInfo>,
 ) -> Result<Box<dyn Hypervisor>> {
     let mem_size = u64::try_from(mgr.shared_mem.mem_size())?;
     let mut regions = mgr.layout.get_memory_regions(&mgr.shared_mem)?;
@@ -915,6 +915,8 @@ fn set_up_hypervisor_partition(
                     pml4_ptr.absolute()?,
                     entrypoint_ptr.absolute()?,
                     rsp_ptr.absolute()?,
+                    #[cfg(gdb)]
+                    debug_info,
                 )?;
                 Ok(Box::new(hv))
             }
diff --git a/src/hyperlight_host/src/hypervisor/kvm.rs b/src/hyperlight_host/src/hypervisor/kvm.rs
index e21c6a35..a86e6ccf 100644
--- a/src/hyperlight_host/src/hypervisor/kvm.rs
+++ b/src/hyperlight_host/src/hypervisor/kvm.rs
@@ -23,6 +23,8 @@ use kvm_ioctls::{Kvm, VcpuExit, VcpuFd, VmFd};
 use tracing::{instrument, Span};
 
 use super::fpu::{FP_CONTROL_WORD_DEFAULT, FP_TAG_WORD_DEFAULT, MXCSR_DEFAULT};
+#[cfg(gdb)]
+use super::gdb::create_gdb_thread;
 use super::handlers::{MemAccessHandlerWrapper, OutBHandlerWrapper};
 use super::{
     HyperlightExit, Hypervisor, VirtualCPU, CR0_AM, CR0_ET, CR0_MP, CR0_NE, CR0_PE, CR0_PG, CR0_WP,
@@ -31,6 +33,8 @@ use super::{
 use crate::hypervisor::hypervisor_handler::HypervisorHandler;
 use crate::mem::memory_region::{MemoryRegion, MemoryRegionFlags};
 use crate::mem::ptr::{GuestPtr, RawPtr};
+#[cfg(gdb)]
+use crate::sandbox::config::DebugInfo;
 use crate::{log_then_return, new_error, Result};
 
 /// Return `true` if the KVM API is available, version 12, and has UserMemory capability, or `false` otherwise
@@ -75,6 +79,7 @@ impl KVMDriver {
         pml4_addr: u64,
         entrypoint: u64,
         rsp: u64,
+        #[cfg(gdb)] debug_info: &Option<DebugInfo>,
     ) -> Result<Self> {
         let kvm = Kvm::new()?;
 
@@ -101,6 +106,11 @@ impl KVMDriver {
         let mut vcpu_fd = vm_fd.create_vcpu(0)?;
         Self::setup_initial_sregs(&mut vcpu_fd, pml4_addr)?;
 
+        #[cfg(gdb)]
+        if let Some(DebugInfo { port }) = debug_info {
+            create_gdb_thread(*port).map_err(|_| new_error!("Cannot create GDB thread"))?;
+        }
+
         let rsp_gp = GuestPtr::try_from(RawPtr::from(rsp))?;
         Ok(Self {
             _kvm: kvm,
diff --git a/src/hyperlight_host/src/hypervisor/mod.rs b/src/hyperlight_host/src/hypervisor/mod.rs
index 62ec8ca8..f7364689 100644
--- a/src/hyperlight_host/src/hypervisor/mod.rs
+++ b/src/hyperlight_host/src/hypervisor/mod.rs
@@ -34,6 +34,10 @@ pub mod hyperv_linux;
 pub(crate) mod hyperv_windows;
 pub(crate) mod hypervisor_handler;
 
+/// GDB debugging support
+#[cfg(gdb)]
+mod gdb;
+
 /// Driver for running in process instead of using hypervisor
 #[cfg(inprocess)]
 pub mod inprocess;

From 431a99bdd0be106ffa445208af6fe0b7ea872b48 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Doru=20Bl=C3=A2nzeanu?= <dblnz@pm.me>
Date: Tue, 4 Feb 2025 19:44:47 +0200
Subject: [PATCH 05/18] add debug communication channel type
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

- this type will be used by the gdb and the hypervisor handler
to send requests and receive responses

Signed-off-by: Doru Blânzeanu <dblnz@pm.me>
---
 src/hyperlight_host/src/hypervisor/gdb/mod.rs | 49 +++++++++++++++++++
 1 file changed, 49 insertions(+)

diff --git a/src/hyperlight_host/src/hypervisor/gdb/mod.rs b/src/hyperlight_host/src/hypervisor/gdb/mod.rs
index 340d41e2..f738025e 100644
--- a/src/hyperlight_host/src/hypervisor/gdb/mod.rs
+++ b/src/hyperlight_host/src/hypervisor/gdb/mod.rs
@@ -14,9 +14,12 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
+#![allow(dead_code)]
 use std::io::{self, ErrorKind};
 use std::net::TcpListener;
 use std::thread;
+
+use crossbeam_channel::{Receiver, Sender, TryRecvError};
 use thiserror::Error;
 #[derive(Debug, Error)]
 pub enum GdbTargetError {
@@ -24,6 +27,10 @@ pub enum GdbTargetError {
     CannotBind,
     #[error("Error encountered while listening for connections")]
     ListenerError,
+    #[error("Error encountered when waiting to receive message")]
+    CannotReceiveMsg,
+    #[error("Error encountered when sending message")]
+    CannotSendMsg,
     #[error("Unexpected error encountered")]
     UnexpectedError,
 }
@@ -40,6 +47,48 @@ impl From<io::Error> for GdbTargetError {
         }
     }
 }
+/// Type that takes care of communication between Hypervisor and Gdb
+pub struct DebugCommChannel<T, U> {
+    /// Transmit channel
+    tx: Sender<T>,
+    /// Receive channel
+    rx: Receiver<U>,
+}
+
+impl<T, U> DebugCommChannel<T, U> {
+    pub fn unbounded() -> (DebugCommChannel<T, U>, DebugCommChannel<U, T>) {
+        let (hyp_tx, gdb_rx): (Sender<U>, Receiver<U>) = crossbeam_channel::unbounded();
+        let (gdb_tx, hyp_rx): (Sender<T>, Receiver<T>) = crossbeam_channel::unbounded();
+
+        let gdb_conn = DebugCommChannel {
+            tx: gdb_tx,
+            rx: gdb_rx,
+        };
+
+        let hyp_conn = DebugCommChannel {
+            tx: hyp_tx,
+            rx: hyp_rx,
+        };
+
+        (gdb_conn, hyp_conn)
+    }
+
+    /// Sends message over the transmit channel and expects a response
+    pub fn send(&self, msg: T) -> Result<(), GdbTargetError> {
+        self.tx.send(msg).map_err(|_| GdbTargetError::CannotSendMsg)
+    }
+
+    /// Waits for a message over the receive channel
+    pub fn recv(&self) -> Result<U, GdbTargetError> {
+        self.rx.recv().map_err(|_| GdbTargetError::CannotReceiveMsg)
+    }
+
+    /// Checks whether there's a message waiting on the receive channel
+    pub fn try_recv(&self) -> Result<U, TryRecvError> {
+        self.rx.try_recv()
+    }
+}
+
 /// Creates a thread that handles gdb protocol
 pub fn create_gdb_thread(
     port: u16,

From 3cf00f7f9f057a2bf1bcd2871038d4bf627b5949 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Doru=20Bl=C3=A2nzeanu?= <dblnz@pm.me>
Date: Tue, 4 Feb 2025 20:17:53 +0200
Subject: [PATCH 06/18] add hyperlight sandbox target to handle gdb commands
 support
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

- the target implements the  traits to provide callbacks
for the gdb commands

Signed-off-by: Doru Blânzeanu <dblnz@pm.me>
---
 .../src/hypervisor/gdb/event_loop.rs          | 117 +++++++++++++++++
 src/hyperlight_host/src/hypervisor/gdb/mod.rs |  65 ++++++++-
 .../src/hypervisor/gdb/x86_64_target.rs       | 124 ++++++++++++++++++
 src/hyperlight_host/src/hypervisor/mod.rs     |  20 +++
 4 files changed, 321 insertions(+), 5 deletions(-)
 create mode 100644 src/hyperlight_host/src/hypervisor/gdb/event_loop.rs
 create mode 100644 src/hyperlight_host/src/hypervisor/gdb/x86_64_target.rs

diff --git a/src/hyperlight_host/src/hypervisor/gdb/event_loop.rs b/src/hyperlight_host/src/hypervisor/gdb/event_loop.rs
new file mode 100644
index 00000000..a1fd6456
--- /dev/null
+++ b/src/hyperlight_host/src/hypervisor/gdb/event_loop.rs
@@ -0,0 +1,117 @@
+/*
+Copyright 2024 The Hyperlight Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+use gdbstub::conn::ConnectionExt;
+use gdbstub::stub::run_blocking::{self, WaitForStopReasonError};
+use gdbstub::stub::{BaseStopReason, DisconnectReason, GdbStub, SingleThreadStopReason};
+
+use super::x86_64_target::HyperlightSandboxTarget;
+use super::{DebugResponse, VcpuStopReason};
+
+pub struct GdbBlockingEventLoop;
+
+impl run_blocking::BlockingEventLoop for GdbBlockingEventLoop {
+    type Connection = Box<dyn ConnectionExt<Error = std::io::Error>>;
+    type StopReason = SingleThreadStopReason<u64>;
+    type Target = HyperlightSandboxTarget;
+
+    fn wait_for_stop_reason(
+        target: &mut Self::Target,
+        conn: &mut Self::Connection,
+    ) -> Result<
+        run_blocking::Event<Self::StopReason>,
+        run_blocking::WaitForStopReasonError<
+            <Self::Target as gdbstub::target::Target>::Error,
+            <Self::Connection as gdbstub::conn::Connection>::Error,
+        >,
+    > {
+        loop {
+            match target.try_recv() {
+                Ok(DebugResponse::VcpuStopped(stop_reason)) => {
+                    log::debug!("VcpuStopped with reason {:?}", stop_reason);
+
+                    // Resume execution if unknown reason for stop
+                    let stop_response = match stop_reason {
+                        VcpuStopReason::DoneStep => BaseStopReason::DoneStep,
+                        VcpuStopReason::SwBp => BaseStopReason::SwBreak(()),
+                        VcpuStopReason::HwBp => BaseStopReason::HwBreak(()),
+                        VcpuStopReason::Unknown => {
+                            target
+                                .resume_vcpu()
+                                .map_err(WaitForStopReasonError::Target)?;
+
+                            continue;
+                        }
+                    };
+
+                    return Ok(run_blocking::Event::TargetStopped(stop_response));
+                }
+                Ok(msg) => {
+                    log::error!("Unexpected message received {:?}", msg);
+                }
+                Err(crossbeam_channel::TryRecvError::Empty) => (),
+                Err(crossbeam_channel::TryRecvError::Disconnected) => {
+                    return Ok(run_blocking::Event::TargetStopped(BaseStopReason::Exited(
+                        0,
+                    )));
+                }
+            }
+
+            // Check if there is any data to read from the connection
+            // If there is, return the data as an incoming data event
+            // Otherwise, continue waiting for a stop reason from the target
+            if conn.peek().map(|b| b.is_some()).unwrap_or(false) {
+                let byte = conn
+                    .read()
+                    .map_err(run_blocking::WaitForStopReasonError::Connection)?;
+
+                return Ok(run_blocking::Event::IncomingData(byte));
+            }
+        }
+    }
+
+    /// Handle an interrupt from the GDB client.
+    /// This function is called when the GDB client sends an interrupt signal.
+    /// Passing `None` defers sending a stop reason to later (e.g. when the target stops).
+    fn on_interrupt(
+        _target: &mut Self::Target,
+    ) -> Result<Option<Self::StopReason>, <Self::Target as gdbstub::target::Target>::Error> {
+        Ok(None)
+    }
+}
+
+pub fn event_loop_thread(
+    debugger: GdbStub<HyperlightSandboxTarget, Box<dyn ConnectionExt<Error = std::io::Error>>>,
+    target: &mut HyperlightSandboxTarget,
+) {
+    match debugger.run_blocking::<GdbBlockingEventLoop>(target) {
+        Ok(disconnect_reason) => match disconnect_reason {
+            DisconnectReason::Disconnect => { 
+                log::info!("Gdb client disconnected");
+            }
+            DisconnectReason::TargetExited(_) => {
+                log::info!("Guest finalized execution and disconnected");
+            }
+            DisconnectReason::TargetTerminated(sig) => {
+                log::info!("Gdb target terminated with signal {}", sig)
+            }
+            DisconnectReason::Kill => log::info!("Gdb sent a kill command"),
+        },
+        Err(e) => {
+            log::error!("fatal error encountered: {e:?}");
+        }
+    }
+}
diff --git a/src/hyperlight_host/src/hypervisor/gdb/mod.rs b/src/hyperlight_host/src/hypervisor/gdb/mod.rs
index f738025e..b7e8a116 100644
--- a/src/hyperlight_host/src/hypervisor/gdb/mod.rs
+++ b/src/hyperlight_host/src/hypervisor/gdb/mod.rs
@@ -14,13 +14,21 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-#![allow(dead_code)]
+mod event_loop;
+pub mod x86_64_target;
+
 use std::io::{self, ErrorKind};
 use std::net::TcpListener;
 use std::thread;
 
 use crossbeam_channel::{Receiver, Sender, TryRecvError};
+use event_loop::event_loop_thread;
+use gdbstub::conn::ConnectionExt;
+use gdbstub::stub::GdbStub;
+use gdbstub::target::TargetError;
 use thiserror::Error;
+use x86_64_target::HyperlightSandboxTarget;
+
 #[derive(Debug, Error)]
 pub enum GdbTargetError {
     #[error("Error encountered while binding to address and port")]
@@ -31,6 +39,8 @@ pub enum GdbTargetError {
     CannotReceiveMsg,
     #[error("Error encountered when sending message")]
     CannotSendMsg,
+    #[error("Encountered an unexpected message over communication channel")]
+    UnexpectedMessage,
     #[error("Unexpected error encountered")]
     UnexpectedError,
 }
@@ -47,7 +57,39 @@ impl From<io::Error> for GdbTargetError {
         }
     }
 }
-/// Type that takes care of communication between Hypervisor and Gdb
+
+impl From<GdbTargetError> for TargetError<GdbTargetError> {
+    fn from(value: GdbTargetError) -> TargetError<GdbTargetError> {
+        TargetError::Io(std::io::Error::other(value))
+    }
+}
+
+/// Defines the possible reasons for which a vCPU can be stopped when debugging
+#[derive(Debug)]
+pub enum VcpuStopReason {
+    DoneStep,
+    HwBp,
+    SwBp,
+    Unknown,
+}
+
+/// Enumerates the possible actions that a debugger can ask from a Hypervisor
+#[allow(dead_code)]
+#[derive(Debug)]
+pub enum DebugMsg {
+    Continue,
+}
+
+/// Enumerates the possible responses that a hypervisor can provide to a debugger
+#[allow(dead_code)]
+#[derive(Debug)]
+pub enum DebugResponse {
+    Continue,
+    VcpuStopped(VcpuStopReason),
+}
+
+/// Debug communication channel that is used for sending a request type and
+/// receive a different response type
 pub struct DebugCommChannel<T, U> {
     /// Transmit channel
     tx: Sender<T>,
@@ -92,7 +134,8 @@ impl<T, U> DebugCommChannel<T, U> {
 /// Creates a thread that handles gdb protocol
 pub fn create_gdb_thread(
     port: u16,
-) -> Result<(), GdbTargetError> {
+) -> Result<DebugCommChannel<DebugResponse, DebugMsg>, GdbTargetError> {
+    let (gdb_conn, hyp_conn) = DebugCommChannel::unbounded();
     let socket = format!("localhost:{}", port);
 
     log::info!("Listening on {:?}", socket);
@@ -103,9 +146,21 @@ pub fn create_gdb_thread(
         .name("GDB handler".to_string())
         .spawn(move || -> Result<(), GdbTargetError> {
             log::info!("Waiting for GDB connection ... ");
-            let (_, _) = listener.accept()?;
+            let (conn, _) = listener.accept()?;
+
+            let conn: Box<dyn ConnectionExt<Error = io::Error>> = Box::new(conn);
+            let debugger = GdbStub::new(conn);
+
+            let mut target = HyperlightSandboxTarget::new(hyp_conn);
+
+            // Waits for vCPU to stop at entrypoint breakpoint
+            let res = target.recv()?;
+            if let DebugResponse::VcpuStopped(_) = res {
+                event_loop_thread(debugger, &mut target);
+            }
+
             Ok(())
         });
 
-    Ok(())
+    Ok(gdb_conn)
 }
diff --git a/src/hyperlight_host/src/hypervisor/gdb/x86_64_target.rs b/src/hyperlight_host/src/hypervisor/gdb/x86_64_target.rs
new file mode 100644
index 00000000..d433186a
--- /dev/null
+++ b/src/hyperlight_host/src/hypervisor/gdb/x86_64_target.rs
@@ -0,0 +1,124 @@
+/*
+Copyright 2024 The Hyperlight Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+use crossbeam_channel::TryRecvError;
+use gdbstub::arch::Arch;
+use gdbstub::target::ext::base::singlethread::SingleThreadBase;
+use gdbstub::target::ext::base::BaseOps;
+use gdbstub::target::{Target, TargetResult};
+use gdbstub_arch::x86::X86_64_SSE as GdbTargetArch;
+
+use super::{DebugMsg, DebugResponse, DebugCommChannel, GdbTargetError};
+
+/// Gdbstub target used by the gdbstub crate to provide GDB protocol implementation
+pub struct HyperlightSandboxTarget {
+    /// Hypervisor communication channels
+    hyp_conn: DebugCommChannel<DebugMsg, DebugResponse>,
+}
+
+impl HyperlightSandboxTarget {
+    pub fn new(hyp_conn: DebugCommChannel<DebugMsg, DebugResponse>) -> Self {
+        HyperlightSandboxTarget { hyp_conn }
+    }
+
+    /// Sends a command over the communication channel and waits for response
+    fn send_command(&self, cmd: DebugMsg) -> Result<DebugResponse, GdbTargetError> {
+        self.send(cmd)?;
+
+        // Wait for response
+        self.recv()
+    }
+
+    /// Sends a command over the communication channel
+    fn send(&self, ev: DebugMsg) -> Result<(), GdbTargetError> {
+        self.hyp_conn.send(ev)
+    }
+
+    /// Waits for a response over the communication channel
+    pub fn recv(&self) -> Result<DebugResponse, GdbTargetError> {
+        self.hyp_conn.recv()
+    }
+
+    /// Non-Blocking check for a response over the communication channel
+    pub fn try_recv(&self) -> Result<DebugResponse, TryRecvError> {
+        self.hyp_conn.try_recv()
+    }
+
+    /// Sends an event to the Hypervisor that tells it to resume vCPU execution
+    /// Note: The method waits for a confirmation message
+    pub fn resume_vcpu(&mut self) -> Result<(), GdbTargetError> {
+        log::info!("Resume vCPU execution");
+
+        match self.send_command(DebugMsg::Continue)? {
+            DebugResponse::Continue => Ok(()),
+            msg => {
+                log::error!("Unexpected message received: {:?}", msg);
+                Err(GdbTargetError::UnexpectedMessage)
+            }
+        }
+    }
+
+}
+
+impl Target for HyperlightSandboxTarget {
+    type Arch = GdbTargetArch;
+    type Error = GdbTargetError;
+
+    #[inline(always)]
+    fn base_ops(&mut self) -> BaseOps<Self::Arch, Self::Error> {
+        BaseOps::SingleThread(self)
+    }
+}
+
+impl SingleThreadBase for HyperlightSandboxTarget {
+    fn read_addrs(
+        &mut self,
+        gva: <Self::Arch as Arch>::Usize,
+        data: &mut [u8],
+    ) -> TargetResult<usize, Self> {
+        log::debug!("Read addr: {:X} len: {:X}", gva, data.len());
+
+        unimplemented!()
+    }
+
+    fn write_addrs(
+        &mut self,
+        gva: <Self::Arch as Arch>::Usize,
+        data: &[u8],
+    ) -> TargetResult<(), Self> {
+        log::debug!("Write addr: {:X} len: {:X}", gva, data.len());
+
+        unimplemented!()
+    }
+
+    fn read_registers(
+        &mut self,
+        _regs: &mut <Self::Arch as Arch>::Registers,
+    ) -> TargetResult<(), Self> {
+        log::debug!("Read regs");
+
+        unimplemented!()
+    }
+
+    fn write_registers(
+        &mut self,
+        _regs: &<Self::Arch as Arch>::Registers,
+    ) -> TargetResult<(), Self> {
+        log::debug!("Write regs");
+
+        unimplemented!()
+    }
+}
diff --git a/src/hyperlight_host/src/hypervisor/mod.rs b/src/hyperlight_host/src/hypervisor/mod.rs
index f7364689..6ca78664 100644
--- a/src/hyperlight_host/src/hypervisor/mod.rs
+++ b/src/hyperlight_host/src/hypervisor/mod.rs
@@ -65,6 +65,9 @@ pub(crate) mod crashdump;
 use std::fmt::Debug;
 use std::sync::{Arc, Mutex};
 
+#[cfg(gdb)]
+use gdb::VcpuStopReason;
+
 use self::handlers::{
     MemAccessHandlerCaller, MemAccessHandlerWrapper, OutBHandlerCaller, OutBHandlerWrapper,
 };
@@ -89,6 +92,9 @@ pub(crate) const EFER_NX: u64 = 1 << 11;
 /// These are the generic exit reasons that we can handle from a Hypervisor the Hypervisors run method is responsible for mapping from
 /// the hypervisor specific exit reasons to these generic ones
 pub enum HyperlightExit {
+    #[cfg(gdb)]
+    /// The vCPU has exited due to a debug event
+    Debug(VcpuStopReason),
     /// The vCPU has halted
     Halt(),
     /// The vCPU has issued a write to the given port with the given value
@@ -193,6 +199,15 @@ pub(crate) trait Hypervisor: Debug + Sync + Send {
 
     #[cfg(crashdump)]
     fn get_memory_regions(&self) -> &[MemoryRegion];
+
+    #[cfg(gdb)]
+    /// handles the cases when the vCPU stops due to a Debug event
+    fn handle_debug(
+        &mut self,
+        _stop_reason: VcpuStopReason,
+    ) -> Result<()> {
+        unimplemented!()
+    }
 }
 
 /// A virtual CPU that can be run until an exit occurs
@@ -209,6 +224,11 @@ impl VirtualCPU {
     ) -> Result<()> {
         loop {
             match hv.run() {
+                #[cfg(gdb)]
+                Ok(HyperlightExit::Debug(stop_reason)) => {
+                    hv.handle_debug(stop_reason)?;
+                }
+
                 Ok(HyperlightExit::Halt()) => {
                     break;
                 }

From 308825addc51399470ce3676961fa47544638949 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Doru=20Bl=C3=A2nzeanu?= <dblnz@pm.me>
Date: Tue, 4 Feb 2025 21:14:06 +0200
Subject: [PATCH 07/18] handle debug vcpu exit on kvm side
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

- adds a specific handler for the vcpu exit debug that waits for
debug messages and processes them

Signed-off-by: Doru Blânzeanu <dblnz@pm.me>
---
 src/hyperlight_host/src/hypervisor/kvm.rs | 111 ++++++++++++++++++++--
 1 file changed, 105 insertions(+), 6 deletions(-)

diff --git a/src/hyperlight_host/src/hypervisor/kvm.rs b/src/hyperlight_host/src/hypervisor/kvm.rs
index a86e6ccf..49c86f3b 100644
--- a/src/hyperlight_host/src/hypervisor/kvm.rs
+++ b/src/hyperlight_host/src/hypervisor/kvm.rs
@@ -24,7 +24,7 @@ use tracing::{instrument, Span};
 
 use super::fpu::{FP_CONTROL_WORD_DEFAULT, FP_TAG_WORD_DEFAULT, MXCSR_DEFAULT};
 #[cfg(gdb)]
-use super::gdb::create_gdb_thread;
+use super::gdb::{create_gdb_thread, DebugMsg, DebugResponse, DebugCommChannel, VcpuStopReason};
 use super::handlers::{MemAccessHandlerWrapper, OutBHandlerWrapper};
 use super::{
     HyperlightExit, Hypervisor, VirtualCPU, CR0_AM, CR0_ET, CR0_MP, CR0_NE, CR0_PE, CR0_PG, CR0_WP,
@@ -59,6 +59,56 @@ pub(crate) fn is_hypervisor_present() -> bool {
     }
 }
 
+#[cfg(gdb)]
+mod debug {
+    use super::KVMDriver;
+    use crate::hypervisor::gdb::{DebugMsg, DebugResponse, VcpuStopReason};
+    use crate::{new_error, Result};
+
+
+    impl KVMDriver {
+        /// Get the reason the vCPU has stopped
+        pub fn get_stop_reason(&self) -> Result<VcpuStopReason> {
+            Ok(VcpuStopReason::Unknown)
+        }
+
+        pub fn process_dbg_request(
+            &mut self,
+            req: DebugMsg,
+        ) -> Result<DebugResponse> {
+            match req {
+                DebugMsg::Continue => {
+                    Ok(DebugResponse::Continue)
+                }
+            }
+        }
+
+        pub fn recv_dbg_msg(&mut self) -> Result<DebugMsg> {
+            let gdb_conn = self
+                .gdb_conn
+                .as_mut()
+                .ok_or_else(|| new_error!("Debug is not enabled"))?;
+
+            gdb_conn
+                .recv()
+                .map_err(|e| new_error!("Got an error while waiting to receive a message: {:?}", e))
+        }
+
+        pub fn send_dbg_msg(&mut self, cmd: DebugResponse) -> Result<()> {
+            log::debug!("Sending {:?}", cmd);
+
+            let gdb_conn = self
+                .gdb_conn
+                .as_mut()
+                .ok_or_else(|| new_error!("Debug is not enabled"))?;
+
+            gdb_conn
+                .send(cmd)
+                .map_err(|e| new_error!("Got an error while sending a response message {:?}", e))
+        }
+    }
+}
+
 /// A Hypervisor driver for KVM on Linux
 pub(super) struct KVMDriver {
     _kvm: Kvm,
@@ -67,6 +117,9 @@ pub(super) struct KVMDriver {
     entrypoint: u64,
     orig_rsp: GuestPtr,
     mem_regions: Vec<MemoryRegion>,
+
+    #[cfg(gdb)]
+    gdb_conn: Option<DebugCommChannel<DebugResponse, DebugMsg>>,
 }
 
 impl KVMDriver {
@@ -107,19 +160,26 @@ impl KVMDriver {
         Self::setup_initial_sregs(&mut vcpu_fd, pml4_addr)?;
 
         #[cfg(gdb)]
-        if let Some(DebugInfo { port }) = debug_info {
-            create_gdb_thread(*port).map_err(|_| new_error!("Cannot create GDB thread"))?;
-        }
+        let gdb_conn = if let Some(DebugInfo { port }) = debug_info {
+                Some(create_gdb_thread(*port).map_err(|_| new_error!("Cannot create GDB thread"))?)
+        } else {
+            None
+        };
 
         let rsp_gp = GuestPtr::try_from(RawPtr::from(rsp))?;
-        Ok(Self {
+
+        let ret = Self {
             _kvm: kvm,
             _vm_fd: vm_fd,
             vcpu_fd,
             entrypoint,
             orig_rsp: rsp_gp,
             mem_regions,
-        })
+            #[cfg(gdb)]
+            gdb_conn,
+        };
+
+        Ok(ret)
     }
 
     #[instrument(err(Debug), skip_all, parent = Span::current(), level = "Trace")]
@@ -311,6 +371,12 @@ impl Hypervisor for KVMDriver {
                     None => HyperlightExit::Mmio(addr),
                 }
             }
+            #[cfg(gdb)]
+            Ok(VcpuExit::Debug(_)) => {
+                let reason = self.get_stop_reason()?;
+
+                HyperlightExit::Debug(reason)
+            }
             Err(e) => match e.errno() {
                 // we send a signal to the thread to cancel execution this results in EINTR being returned by KVM so we return Cancelled
                 libc::EINTR => HyperlightExit::Cancelled(),
@@ -337,7 +403,40 @@ impl Hypervisor for KVMDriver {
     fn get_memory_regions(&self) -> &[MemoryRegion] {
         &self.mem_regions
     }
+
+    #[cfg(gdb)]
+    fn handle_debug(
+        &mut self,
+        stop_reason: VcpuStopReason,
+    ) -> Result<()> {
+        self.send_dbg_msg(DebugResponse::VcpuStopped(stop_reason))
+            .map_err(|e| new_error!("Couldn't signal vCPU stopped event to GDB thread: {:?}", e))?;
+
+        loop {
+            log::debug!("Debug wait for event to resume vCPU");
+            // Wait for a message from gdb
+            let req = self.recv_dbg_msg()?;
+
+            let response = self.process_dbg_request(req)?;
+
+            // If the command was either step or continue, we need to run the vcpu
+            let cont = matches!(
+                response,
+                DebugResponse::Continue
+            );
+
+            self.send_dbg_msg(response)
+                .map_err(|e| new_error!("Couldn't send response to gdb: {:?}", e))?;
+
+            if cont {
+                break;
+            }
+        }
+
+        Ok(())
+    }
 }
+
 #[cfg(test)]
 mod tests {
     use std::sync::{Arc, Mutex};

From b4cb43d9c4a11b2e7f49e6437d88b51495569892 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Doru=20Bl=C3=A2nzeanu?= <dblnz@pm.me>
Date: Tue, 4 Feb 2025 21:28:38 +0200
Subject: [PATCH 08/18] add kvm debug configuration that handles vCPU
 interraction
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Signed-off-by: Doru Blânzeanu <dblnz@pm.me>
---
 src/hyperlight_host/src/hypervisor/kvm.rs | 206 +++++++++++++++++++++-
 1 file changed, 202 insertions(+), 4 deletions(-)

diff --git a/src/hyperlight_host/src/hypervisor/kvm.rs b/src/hyperlight_host/src/hypervisor/kvm.rs
index 49c86f3b..1a0e93e9 100644
--- a/src/hyperlight_host/src/hypervisor/kvm.rs
+++ b/src/hyperlight_host/src/hypervisor/kvm.rs
@@ -61,14 +61,190 @@ pub(crate) fn is_hypervisor_present() -> bool {
 
 #[cfg(gdb)]
 mod debug {
+    use kvm_bindings::{
+        kvm_guest_debug, KVM_GUESTDBG_ENABLE, KVM_GUESTDBG_SINGLESTEP,
+        KVM_GUESTDBG_USE_HW_BP, KVM_GUESTDBG_USE_SW_BP,
+    };
+    use kvm_ioctls::VcpuFd;
+
     use super::KVMDriver;
     use crate::hypervisor::gdb::{DebugMsg, DebugResponse, VcpuStopReason};
     use crate::{new_error, Result};
 
+    /// KVM Debug struct
+    /// This struct is used to abstract the internal details of the kvm
+    /// guest debugging settings
+    pub struct KvmDebug {
+        /// vCPU stepping state
+        single_step: bool,
+
+        /// Array of addresses for HW breakpoints
+        hw_breakpoints: Vec<u64>,
+
+        /// Sent to KVM for enabling guest debug
+        pub dbg_cfg: kvm_guest_debug,
+    }
+
+    impl KvmDebug {
+        const MAX_NO_OF_HW_BP: usize = 4;
+
+        pub fn new() -> Self {
+            let dbg = kvm_guest_debug {
+                control: KVM_GUESTDBG_ENABLE | KVM_GUESTDBG_USE_SW_BP,
+                ..Default::default()
+            };
+
+            Self {
+                single_step: false,
+                hw_breakpoints: vec![],
+                dbg_cfg: dbg,
+            }
+        }
+
+        /// This method sets the kvm debugreg fields to enable breakpoints at
+        /// specific addresses
+        ///
+        /// The first 4 debug registers are used to set the addresses
+        /// The 4th and 5th debug registers are obsolete and not used
+        /// The 7th debug register is used to enable the breakpoints
+        /// For more information see: https://en.wikipedia.org/wiki/X86_debug_register
+        fn set_debug_config(&mut self, vcpu_fd: &VcpuFd, step: bool) -> Result<()> {
+            let addrs = &self.hw_breakpoints;
+
+            self.dbg_cfg.arch.debugreg = [0; 8];
+            for (k, addr) in addrs.iter().enumerate() {
+                self.dbg_cfg.arch.debugreg[k] = *addr;
+                self.dbg_cfg.arch.debugreg[7] |= 1 << (k * 2);
+            }
+
+            if !addrs.is_empty() {
+                self.dbg_cfg.control |= KVM_GUESTDBG_USE_HW_BP;
+            } else {
+                self.dbg_cfg.control &= !KVM_GUESTDBG_USE_HW_BP;
+            }
+
+            if step {
+                self.dbg_cfg.control |= KVM_GUESTDBG_SINGLESTEP;
+            } else {
+                self.dbg_cfg.control &= !KVM_GUESTDBG_SINGLESTEP;
+            }
+
+            log::debug!("Setting bp: {:?} cfg: {:?}", addrs, self.dbg_cfg);
+            vcpu_fd
+                .set_guest_debug(&self.dbg_cfg)
+                .map_err(|e| new_error!("Could not set guest debug: {:?}", e))?;
+
+            self.single_step = step;
+
+            Ok(())
+        }
+
+        /// Method that adds a breakpoint
+        fn add_breakpoint(&mut self, vcpu_fd: &VcpuFd, addr: u64) -> Result<bool> {
+            if self.hw_breakpoints.len() >= Self::MAX_NO_OF_HW_BP {
+                Ok(false)
+            } else if self.hw_breakpoints.contains(&addr) {
+                Ok(true)
+            } else {
+                self.hw_breakpoints.push(addr);
+                self.set_debug_config(vcpu_fd, self.single_step)?;
+
+                Ok(true)
+            }
+        }
+
+        /// Method that removes a breakpoint
+        fn remove_breakpoint(&mut self, vcpu_fd: &VcpuFd, addr: u64) -> Result<bool> {
+            if self.hw_breakpoints.contains(&addr) {
+                self.hw_breakpoints.retain(|&a| a != addr);
+                self.set_debug_config(vcpu_fd, self.single_step)?;
+
+                Ok(true)
+            } else {
+                Ok(false)
+            }
+        }
+    }
 
     impl KVMDriver {
+
+        /// Returns the instruction pointer from the stopped vCPU
+        fn get_instruction_pointer(&self) -> Result<u64> {
+            let regs = self
+                .vcpu_fd
+                .get_regs()
+                .map_err(|e| new_error!("Could not retrieve registers from vCPU: {:?}", e))?;
+
+            Ok(regs.rip)
+        }
+
+        /// Sets or clears stepping for vCPU
+        fn set_single_step(&mut self, enable: bool) -> Result<()> {
+            let debug = self
+                .debug
+                .as_mut()
+                .ok_or_else(|| new_error!("Debug is not enabled"))?;
+
+            debug.set_debug_config(&self.vcpu_fd, enable)
+        }
+
+        /// Translates the guest address to physical address
+        fn translate_gva(&self, gva: u64) -> Result<u64> {
+            let tr = self.vcpu_fd.translate_gva(gva).map_err(|e| {
+                new_error!(
+                    "Could not translate guest virtual address {:X}: {:?}",
+                    gva,
+                    e
+                )
+            })?;
+
+            if tr.valid == 0 {
+                Err(new_error!(
+                    "Could not translate guest virtual address {:X}",
+                    gva
+                ))
+            } else {
+                Ok(tr.physical_address)
+            }
+        }
+
+        /// Gdb expects the target to be stopped when connected.
+        /// This method provides a way to set a breakpoint at the entry point
+        /// it does not keep this breakpoint set after the vCPU already stopped at the address
+        pub fn set_entrypoint_bp(&self) -> Result<()> {
+            if self.debug.is_some() {
+                log::debug!("Setting entrypoint bp {:X}", self.entrypoint);
+                let mut entrypoint_debug = KvmDebug::new();
+                entrypoint_debug.add_breakpoint(&self.vcpu_fd, self.entrypoint)?;
+
+                Ok(())
+            } else {
+                Ok(())
+            }
+        }
+
         /// Get the reason the vCPU has stopped
         pub fn get_stop_reason(&self) -> Result<VcpuStopReason> {
+            let debug = self
+                .debug
+                .as_ref()
+                .ok_or_else(|| new_error!("Debug is not enabled"))?;
+
+            if debug.single_step {
+                return Ok(VcpuStopReason::DoneStep);
+            }
+
+            let ip = self.get_instruction_pointer()?;
+            let gpa = self.translate_gva(ip)?;
+
+            if debug.hw_breakpoints.contains(&gpa) {
+                return Ok(VcpuStopReason::HwBp);
+            }
+
+            if ip == self.entrypoint {
+                return Ok(VcpuStopReason::HwBp);
+            }
+
             Ok(VcpuStopReason::Unknown)
         }
 
@@ -78,6 +254,7 @@ mod debug {
         ) -> Result<DebugResponse> {
             match req {
                 DebugMsg::Continue => {
+                    self.set_single_step(false)?;
                     Ok(DebugResponse::Continue)
                 }
             }
@@ -118,6 +295,8 @@ pub(super) struct KVMDriver {
     orig_rsp: GuestPtr,
     mem_regions: Vec<MemoryRegion>,
 
+    #[cfg(gdb)]
+    debug: Option<debug::KvmDebug>,
     #[cfg(gdb)]
     gdb_conn: Option<DebugCommChannel<DebugResponse, DebugMsg>>,
 }
@@ -160,10 +339,23 @@ impl KVMDriver {
         Self::setup_initial_sregs(&mut vcpu_fd, pml4_addr)?;
 
         #[cfg(gdb)]
-        let gdb_conn = if let Some(DebugInfo { port }) = debug_info {
-                Some(create_gdb_thread(*port).map_err(|_| new_error!("Cannot create GDB thread"))?)
-        } else {
-            None
+        let (debug, gdb_conn) = {
+            if let Some(DebugInfo { port }) = debug_info {
+                let gdb_conn = create_gdb_thread(*port);
+
+                // in case the gdb thread creation fails, we still want to continue
+                // without gdb
+                match gdb_conn {
+                    Ok(gdb_conn) => (Some(debug::KvmDebug::new()), Some(gdb_conn)),
+                    Err(e) => {
+                        log::error!("Could not create gdb connection: {:#}", e);
+
+                        (None, None)
+                    }
+                }
+            } else {
+                (None, None)
+            }
         };
 
         let rsp_gp = GuestPtr::try_from(RawPtr::from(rsp))?;
@@ -175,10 +367,16 @@ impl KVMDriver {
             entrypoint,
             orig_rsp: rsp_gp,
             mem_regions,
+
+            #[cfg(gdb)]
+            debug,
             #[cfg(gdb)]
             gdb_conn,
         };
 
+        #[cfg(gdb)]
+        ret.set_entrypoint_bp()?;
+
         Ok(ret)
     }
 

From 3752649c79861ff747c46e515ea7cbc701b836d2 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Doru=20Bl=C3=A2nzeanu?= <dblnz@pm.me>
Date: Tue, 4 Feb 2025 21:48:27 +0200
Subject: [PATCH 09/18] add read/write registers support
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Signed-off-by: Doru Blânzeanu <dblnz@pm.me>
---
 src/hyperlight_host/src/hypervisor/gdb/mod.rs | 29 +++++++-
 .../src/hypervisor/gdb/x86_64_target.rs       | 67 +++++++++++++++--
 src/hyperlight_host/src/hypervisor/kvm.rs     | 72 ++++++++++++++++++-
 3 files changed, 158 insertions(+), 10 deletions(-)

diff --git a/src/hyperlight_host/src/hypervisor/gdb/mod.rs b/src/hyperlight_host/src/hypervisor/gdb/mod.rs
index b7e8a116..e1ed6bdd 100644
--- a/src/hyperlight_host/src/hypervisor/gdb/mod.rs
+++ b/src/hyperlight_host/src/hypervisor/gdb/mod.rs
@@ -64,6 +64,29 @@ impl From<GdbTargetError> for TargetError<GdbTargetError> {
     }
 }
 
+/// Struct that contains the x86_64 core registers
+#[derive(Debug, Default)]
+pub struct X86_64Regs {
+    pub rax: u64,
+    pub rbx: u64,
+    pub rcx: u64,
+    pub rdx: u64,
+    pub rsi: u64,
+    pub rdi: u64,
+    pub rbp: u64,
+    pub rsp: u64,
+    pub r8: u64,
+    pub r9: u64,
+    pub r10: u64,
+    pub r11: u64,
+    pub r12: u64,
+    pub r13: u64,
+    pub r14: u64,
+    pub r15: u64,
+    pub rip: u64,
+    pub rflags: u64,
+}
+
 /// Defines the possible reasons for which a vCPU can be stopped when debugging
 #[derive(Debug)]
 pub enum VcpuStopReason {
@@ -74,18 +97,20 @@ pub enum VcpuStopReason {
 }
 
 /// Enumerates the possible actions that a debugger can ask from a Hypervisor
-#[allow(dead_code)]
 #[derive(Debug)]
 pub enum DebugMsg {
     Continue,
+    ReadRegisters,
+    WriteRegisters(X86_64Regs),
 }
 
 /// Enumerates the possible responses that a hypervisor can provide to a debugger
-#[allow(dead_code)]
 #[derive(Debug)]
 pub enum DebugResponse {
     Continue,
+    ReadRegisters(X86_64Regs),
     VcpuStopped(VcpuStopReason),
+    WriteRegisters,
 }
 
 /// Debug communication channel that is used for sending a request type and
diff --git a/src/hyperlight_host/src/hypervisor/gdb/x86_64_target.rs b/src/hyperlight_host/src/hypervisor/gdb/x86_64_target.rs
index d433186a..b9850e62 100644
--- a/src/hyperlight_host/src/hypervisor/gdb/x86_64_target.rs
+++ b/src/hyperlight_host/src/hypervisor/gdb/x86_64_target.rs
@@ -18,10 +18,10 @@ use crossbeam_channel::TryRecvError;
 use gdbstub::arch::Arch;
 use gdbstub::target::ext::base::singlethread::SingleThreadBase;
 use gdbstub::target::ext::base::BaseOps;
-use gdbstub::target::{Target, TargetResult};
+use gdbstub::target::{Target, TargetError, TargetResult};
 use gdbstub_arch::x86::X86_64_SSE as GdbTargetArch;
 
-use super::{DebugMsg, DebugResponse, DebugCommChannel, GdbTargetError};
+use super::{DebugMsg, DebugResponse, DebugCommChannel, GdbTargetError, X86_64Regs};
 
 /// Gdbstub target used by the gdbstub crate to provide GDB protocol implementation
 pub struct HyperlightSandboxTarget {
@@ -106,19 +106,74 @@ impl SingleThreadBase for HyperlightSandboxTarget {
 
     fn read_registers(
         &mut self,
-        _regs: &mut <Self::Arch as Arch>::Registers,
+        regs: &mut <Self::Arch as Arch>::Registers,
     ) -> TargetResult<(), Self> {
         log::debug!("Read regs");
 
-        unimplemented!()
+        match self.send_command(DebugMsg::ReadRegisters)? {
+            DebugResponse::ReadRegisters(read_regs) => {
+                regs.regs[0] = read_regs.rax;
+                regs.regs[1] = read_regs.rbp;
+                regs.regs[2] = read_regs.rcx;
+                regs.regs[3] = read_regs.rdx;
+                regs.regs[4] = read_regs.rsi;
+                regs.regs[5] = read_regs.rdi;
+                regs.regs[6] = read_regs.rbp;
+                regs.regs[7] = read_regs.rsp;
+                regs.regs[8] = read_regs.r8;
+                regs.regs[9] = read_regs.r9;
+                regs.regs[10] = read_regs.r10;
+                regs.regs[11] = read_regs.r11;
+                regs.regs[12] = read_regs.r12;
+                regs.regs[13] = read_regs.r13;
+                regs.regs[14] = read_regs.r14;
+                regs.regs[15] = read_regs.r15;
+                regs.rip = read_regs.rip;
+                regs.eflags = read_regs.rflags as u32;
+
+                Ok(())
+            }
+
+            msg => {
+                log::error!("Unexpected message received: {:?}", msg);
+                Err(TargetError::Fatal(GdbTargetError::UnexpectedMessage))
+            }
+        }
     }
 
     fn write_registers(
         &mut self,
-        _regs: &<Self::Arch as Arch>::Registers,
+        regs: &<Self::Arch as Arch>::Registers,
     ) -> TargetResult<(), Self> {
         log::debug!("Write regs");
 
-        unimplemented!()
+        let regs = X86_64Regs {
+            rax: regs.regs[0],
+            rbx: regs.regs[1],
+            rcx: regs.regs[2],
+            rdx: regs.regs[3],
+            rsi: regs.regs[4],
+            rdi: regs.regs[5],
+            rbp: regs.regs[6],
+            rsp: regs.regs[7],
+            r8: regs.regs[8],
+            r9: regs.regs[9],
+            r10: regs.regs[10],
+            r11: regs.regs[11],
+            r12: regs.regs[12],
+            r13: regs.regs[13],
+            r14: regs.regs[14],
+            r15: regs.regs[15],
+            rip: regs.rip,
+            rflags: u64::from(regs.eflags),
+        };
+
+        match self.send_command(DebugMsg::WriteRegisters(regs))? {
+            DebugResponse::WriteRegisters => Ok(()),
+            msg => {
+                log::error!("Unexpected message received: {:?}", msg);
+                Err(TargetError::Fatal(GdbTargetError::UnexpectedMessage))
+            }
+        }
     }
 }
diff --git a/src/hyperlight_host/src/hypervisor/kvm.rs b/src/hyperlight_host/src/hypervisor/kvm.rs
index 1a0e93e9..494aaa59 100644
--- a/src/hyperlight_host/src/hypervisor/kvm.rs
+++ b/src/hyperlight_host/src/hypervisor/kvm.rs
@@ -62,13 +62,13 @@ pub(crate) fn is_hypervisor_present() -> bool {
 #[cfg(gdb)]
 mod debug {
     use kvm_bindings::{
-        kvm_guest_debug, KVM_GUESTDBG_ENABLE, KVM_GUESTDBG_SINGLESTEP,
+        kvm_guest_debug, kvm_regs, KVM_GUESTDBG_ENABLE, KVM_GUESTDBG_SINGLESTEP,
         KVM_GUESTDBG_USE_HW_BP, KVM_GUESTDBG_USE_SW_BP,
     };
     use kvm_ioctls::VcpuFd;
 
     use super::KVMDriver;
-    use crate::hypervisor::gdb::{DebugMsg, DebugResponse, VcpuStopReason};
+    use crate::hypervisor::gdb::{DebugMsg, DebugResponse, VcpuStopReason, X86_64Regs};
     use crate::{new_error, Result};
 
     /// KVM Debug struct
@@ -208,6 +208,65 @@ mod debug {
             }
         }
 
+        fn read_regs(&self, regs: &mut X86_64Regs) -> Result<()> {
+            log::debug!("Read registers");
+            let vcpu_regs = self
+                .vcpu_fd
+                .get_regs()
+                .map_err(|e| new_error!("Could not read guest registers: {:?}", e))?;
+
+            regs.rax = vcpu_regs.rax;
+            regs.rbx = vcpu_regs.rbx;
+            regs.rcx = vcpu_regs.rcx;
+            regs.rdx = vcpu_regs.rdx;
+            regs.rsi = vcpu_regs.rsi;
+            regs.rdi = vcpu_regs.rdi;
+            regs.rbp = vcpu_regs.rbp;
+            regs.rsp = vcpu_regs.rsp;
+            regs.r8 = vcpu_regs.r8;
+            regs.r9 = vcpu_regs.r9;
+            regs.r10 = vcpu_regs.r10;
+            regs.r11 = vcpu_regs.r11;
+            regs.r12 = vcpu_regs.r12;
+            regs.r13 = vcpu_regs.r13;
+            regs.r14 = vcpu_regs.r14;
+            regs.r15 = vcpu_regs.r15;
+
+            regs.rip = vcpu_regs.rip;
+            regs.rflags = vcpu_regs.rflags;
+
+            Ok(())
+        }
+
+        fn write_regs(&self, regs: &X86_64Regs) -> Result<()> {
+            log::debug!("Write registers");
+            let new_regs = kvm_regs {
+                rax: regs.rax,
+                rbx: regs.rbx,
+                rcx: regs.rcx,
+                rdx: regs.rdx,
+                rsi: regs.rsi,
+                rdi: regs.rdi,
+                rbp: regs.rbp,
+                rsp: regs.rsp,
+                r8: regs.r8,
+                r9: regs.r9,
+                r10: regs.r10,
+                r11: regs.r11,
+                r12: regs.r12,
+                r13: regs.r13,
+                r14: regs.r14,
+                r15: regs.r15,
+
+                rip: regs.rip,
+                rflags: regs.rflags,
+            };
+
+            self.vcpu_fd
+                .set_regs(&new_regs)
+                .map_err(|e| new_error!("Could not write guest registers: {:?}", e))
+        }
+
         /// Gdb expects the target to be stopped when connected.
         /// This method provides a way to set a breakpoint at the entry point
         /// it does not keep this breakpoint set after the vCPU already stopped at the address
@@ -257,6 +316,15 @@ mod debug {
                     self.set_single_step(false)?;
                     Ok(DebugResponse::Continue)
                 }
+                DebugMsg::ReadRegisters => {
+                    let mut regs = X86_64Regs::default();
+                    self.read_regs(&mut regs).expect("Read Regs error");
+                    Ok(DebugResponse::ReadRegisters(regs))
+                }
+                DebugMsg::WriteRegisters(regs) => {
+                    self.write_regs(&regs).expect("Write Regs error");
+                    Ok(DebugResponse::WriteRegisters)
+                }
             }
         }
 

From a6cce6b21493da5bf699eb48a2bbb634ad4d0a1b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Doru=20Bl=C3=A2nzeanu?= <dblnz@pm.me>
Date: Tue, 4 Feb 2025 21:56:46 +0200
Subject: [PATCH 10/18] hw breakpoint and resume support added
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Signed-off-by: Doru Blânzeanu <dblnz@pm.me>
---
 src/hyperlight_host/src/hypervisor/gdb/mod.rs |  4 ++
 .../src/hypervisor/gdb/x86_64_target.rs       | 63 ++++++++++++++++++-
 src/hyperlight_host/src/hypervisor/kvm.rs     | 32 ++++++++++
 3 files changed, 98 insertions(+), 1 deletion(-)

diff --git a/src/hyperlight_host/src/hypervisor/gdb/mod.rs b/src/hyperlight_host/src/hypervisor/gdb/mod.rs
index e1ed6bdd..f9ea59af 100644
--- a/src/hyperlight_host/src/hypervisor/gdb/mod.rs
+++ b/src/hyperlight_host/src/hypervisor/gdb/mod.rs
@@ -99,16 +99,20 @@ pub enum VcpuStopReason {
 /// Enumerates the possible actions that a debugger can ask from a Hypervisor
 #[derive(Debug)]
 pub enum DebugMsg {
+    AddHwBreakpoint(u64),
     Continue,
     ReadRegisters,
+    RemoveHwBreakpoint(u64),
     WriteRegisters(X86_64Regs),
 }
 
 /// Enumerates the possible responses that a hypervisor can provide to a debugger
 #[derive(Debug)]
 pub enum DebugResponse {
+    AddHwBreakpoint(bool),
     Continue,
     ReadRegisters(X86_64Regs),
+    RemoveHwBreakpoint(bool),
     VcpuStopped(VcpuStopReason),
     WriteRegisters,
 }
diff --git a/src/hyperlight_host/src/hypervisor/gdb/x86_64_target.rs b/src/hyperlight_host/src/hypervisor/gdb/x86_64_target.rs
index b9850e62..78e9cd69 100644
--- a/src/hyperlight_host/src/hypervisor/gdb/x86_64_target.rs
+++ b/src/hyperlight_host/src/hypervisor/gdb/x86_64_target.rs
@@ -16,8 +16,14 @@ limitations under the License.
 
 use crossbeam_channel::TryRecvError;
 use gdbstub::arch::Arch;
-use gdbstub::target::ext::base::singlethread::SingleThreadBase;
+use gdbstub::common::Signal;
+use gdbstub::target::ext::base::singlethread::{
+    SingleThreadBase, SingleThreadResume, SingleThreadResumeOps,
+};
 use gdbstub::target::ext::base::BaseOps;
+use gdbstub::target::ext::breakpoints::{
+    Breakpoints, BreakpointsOps, HwBreakpoint, HwBreakpointOps,
+};
 use gdbstub::target::{Target, TargetError, TargetResult};
 use gdbstub_arch::x86::X86_64_SSE as GdbTargetArch;
 
@@ -77,6 +83,10 @@ impl Target for HyperlightSandboxTarget {
     type Arch = GdbTargetArch;
     type Error = GdbTargetError;
 
+    fn support_breakpoints(&mut self) -> Option<BreakpointsOps<Self>> {
+        Some(self)
+    }
+
     #[inline(always)]
     fn base_ops(&mut self) -> BaseOps<Self::Arch, Self::Error> {
         BaseOps::SingleThread(self)
@@ -176,4 +186,55 @@ impl SingleThreadBase for HyperlightSandboxTarget {
             }
         }
     }
+
+    fn support_resume(&mut self) -> Option<SingleThreadResumeOps<Self>> {
+        Some(self)
+    }
+}
+
+impl Breakpoints for HyperlightSandboxTarget {
+    fn support_hw_breakpoint(&mut self) -> Option<HwBreakpointOps<Self>> {
+        Some(self)
+    }
+}
+
+impl HwBreakpoint for HyperlightSandboxTarget {
+    fn add_hw_breakpoint(
+        &mut self,
+        addr: <Self::Arch as Arch>::Usize,
+        _kind: <Self::Arch as Arch>::BreakpointKind,
+    ) -> TargetResult<bool, Self> {
+        log::debug!("Add hw breakpoint at address {:X}", addr);
+
+        match self.send_command(DebugMsg::AddHwBreakpoint(addr))? {
+            DebugResponse::AddHwBreakpoint(rsp) => Ok(rsp),
+            msg => {
+                log::error!("Unexpected message received: {:?}", msg);
+                Err(TargetError::Fatal(GdbTargetError::UnexpectedMessage))
+            }
+        }
+    }
+
+    fn remove_hw_breakpoint(
+        &mut self,
+        addr: <Self::Arch as Arch>::Usize,
+        _kind: <Self::Arch as Arch>::BreakpointKind,
+    ) -> TargetResult<bool, Self> {
+        log::debug!("Remove hw breakpoint at address {:X}", addr);
+
+        match self.send_command(DebugMsg::RemoveHwBreakpoint(addr))? {
+            DebugResponse::RemoveHwBreakpoint(rsp) => Ok(rsp),
+            msg => {
+                log::error!("Unexpected message received: {:?}", msg);
+                Err(TargetError::Fatal(GdbTargetError::UnexpectedMessage))
+            }
+        }
+    }
+}
+
+impl SingleThreadResume for HyperlightSandboxTarget {
+    fn resume(&mut self, _signal: Option<Signal>) -> Result<(), Self::Error> {
+        log::debug!("Resume");
+        self.resume_vcpu()
+    }
 }
diff --git a/src/hyperlight_host/src/hypervisor/kvm.rs b/src/hyperlight_host/src/hypervisor/kvm.rs
index 494aaa59..6abeb44a 100644
--- a/src/hyperlight_host/src/hypervisor/kvm.rs
+++ b/src/hyperlight_host/src/hypervisor/kvm.rs
@@ -267,6 +267,26 @@ mod debug {
                 .map_err(|e| new_error!("Could not write guest registers: {:?}", e))
         }
 
+        fn add_hw_breakpoint(&mut self, addr: u64) -> Result<bool> {
+            let addr = self.translate_gva(addr)?;
+
+            if let Some(debug) = self.debug.as_mut() {
+                debug.add_breakpoint(&self.vcpu_fd, addr)
+            } else {
+                Ok(false)
+            }
+        }
+
+        fn remove_hw_breakpoint(&mut self, addr: u64) -> Result<bool> {
+            let addr = self.translate_gva(addr)?;
+
+            if let Some(debug) = self.debug.as_mut() {
+                debug.remove_breakpoint(&self.vcpu_fd, addr)
+            } else {
+                Ok(false)
+            }
+        }
+
         /// Gdb expects the target to be stopped when connected.
         /// This method provides a way to set a breakpoint at the entry point
         /// it does not keep this breakpoint set after the vCPU already stopped at the address
@@ -312,6 +332,12 @@ mod debug {
             req: DebugMsg,
         ) -> Result<DebugResponse> {
             match req {
+                DebugMsg::AddHwBreakpoint(addr) => {
+                    let res = self
+                        .add_hw_breakpoint(addr)
+                        .expect("Add hw breakpoint error");
+                    Ok(DebugResponse::AddHwBreakpoint(res))
+                }
                 DebugMsg::Continue => {
                     self.set_single_step(false)?;
                     Ok(DebugResponse::Continue)
@@ -321,6 +347,12 @@ mod debug {
                     self.read_regs(&mut regs).expect("Read Regs error");
                     Ok(DebugResponse::ReadRegisters(regs))
                 }
+                DebugMsg::RemoveHwBreakpoint(addr) => {
+                    let res = self
+                        .remove_hw_breakpoint(addr)
+                        .expect("Remove hw breakpoint error");
+                    Ok(DebugResponse::RemoveHwBreakpoint(res))
+                }
                 DebugMsg::WriteRegisters(regs) => {
                     self.write_regs(&regs).expect("Write Regs error");
                     Ok(DebugResponse::WriteRegisters)

From 4860fb6205fdd8c9cb4a4481674c92a62dd49e57 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Doru=20Bl=C3=A2nzeanu?= <dblnz@pm.me>
Date: Tue, 4 Feb 2025 22:14:36 +0200
Subject: [PATCH 11/18] add stepping support
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

- also adds handling of gdb client disconnect by resuming execution

Signed-off-by: Doru Blânzeanu <dblnz@pm.me>
---
 .../src/hypervisor/gdb/event_loop.rs          |  3 ++
 src/hyperlight_host/src/hypervisor/gdb/mod.rs |  4 +++
 .../src/hypervisor/gdb/x86_64_target.rs       | 35 ++++++++++++++++++-
 src/hyperlight_host/src/hypervisor/kvm.rs     | 18 +++++++++-
 4 files changed, 58 insertions(+), 2 deletions(-)

diff --git a/src/hyperlight_host/src/hypervisor/gdb/event_loop.rs b/src/hyperlight_host/src/hypervisor/gdb/event_loop.rs
index a1fd6456..af46c3ae 100644
--- a/src/hyperlight_host/src/hypervisor/gdb/event_loop.rs
+++ b/src/hyperlight_host/src/hypervisor/gdb/event_loop.rs
@@ -101,6 +101,9 @@ pub fn event_loop_thread(
         Ok(disconnect_reason) => match disconnect_reason {
             DisconnectReason::Disconnect => { 
                 log::info!("Gdb client disconnected");
+                if let Err(e) = target.disable_debug() {
+                    log::error!("Cannot disable debugging: {:?}", e);
+                }
             }
             DisconnectReason::TargetExited(_) => {
                 log::info!("Guest finalized execution and disconnected");
diff --git a/src/hyperlight_host/src/hypervisor/gdb/mod.rs b/src/hyperlight_host/src/hypervisor/gdb/mod.rs
index f9ea59af..4570f08f 100644
--- a/src/hyperlight_host/src/hypervisor/gdb/mod.rs
+++ b/src/hyperlight_host/src/hypervisor/gdb/mod.rs
@@ -101,8 +101,10 @@ pub enum VcpuStopReason {
 pub enum DebugMsg {
     AddHwBreakpoint(u64),
     Continue,
+    DisableDebug,
     ReadRegisters,
     RemoveHwBreakpoint(u64),
+    Step,
     WriteRegisters(X86_64Regs),
 }
 
@@ -111,8 +113,10 @@ pub enum DebugMsg {
 pub enum DebugResponse {
     AddHwBreakpoint(bool),
     Continue,
+    DisableDebug,
     ReadRegisters(X86_64Regs),
     RemoveHwBreakpoint(bool),
+    Step,
     VcpuStopped(VcpuStopReason),
     WriteRegisters,
 }
diff --git a/src/hyperlight_host/src/hypervisor/gdb/x86_64_target.rs b/src/hyperlight_host/src/hypervisor/gdb/x86_64_target.rs
index 78e9cd69..70fd43ab 100644
--- a/src/hyperlight_host/src/hypervisor/gdb/x86_64_target.rs
+++ b/src/hyperlight_host/src/hypervisor/gdb/x86_64_target.rs
@@ -18,7 +18,8 @@ use crossbeam_channel::TryRecvError;
 use gdbstub::arch::Arch;
 use gdbstub::common::Signal;
 use gdbstub::target::ext::base::singlethread::{
-    SingleThreadBase, SingleThreadResume, SingleThreadResumeOps,
+    SingleThreadBase, SingleThreadResume, SingleThreadResumeOps, SingleThreadSingleStep,
+    SingleThreadSingleStepOps,
 };
 use gdbstub::target::ext::base::BaseOps;
 use gdbstub::target::ext::breakpoints::{
@@ -77,6 +78,20 @@ impl HyperlightSandboxTarget {
         }
     }
 
+    /// Sends an event to the Hypervisor that tells it to disable debugging
+    /// and continue executing until end
+    /// Note: The method waits for a confirmation message
+    pub fn disable_debug(&mut self) -> Result<(), GdbTargetError> {
+        log::info!("Disable debugging and continue until end");
+
+        match self.send_command(DebugMsg::DisableDebug)? {
+            DebugResponse::DisableDebug => Ok(()),
+            msg => {
+                log::error!("Unexpected message received: {:?}", msg);
+                Err(GdbTargetError::UnexpectedMessage)
+            }
+        }
+    }
 }
 
 impl Target for HyperlightSandboxTarget {
@@ -237,4 +252,22 @@ impl SingleThreadResume for HyperlightSandboxTarget {
         log::debug!("Resume");
         self.resume_vcpu()
     }
+    fn support_single_step(&mut self) -> Option<SingleThreadSingleStepOps<Self>> {
+        Some(self)
+    }
+}
+
+impl SingleThreadSingleStep for HyperlightSandboxTarget {
+    fn step(&mut self, signal: Option<Signal>) -> Result<(), Self::Error> {
+        assert!(signal.is_none());
+
+        log::debug!("Step");
+        match self.send_command(DebugMsg::Step)? {
+            DebugResponse::Step => Ok(()),
+            msg => {
+                log::error!("Unexpected message received: {:?}", msg);
+                Err(GdbTargetError::UnexpectedMessage)
+            }
+        }
+    }
 }
diff --git a/src/hyperlight_host/src/hypervisor/kvm.rs b/src/hyperlight_host/src/hypervisor/kvm.rs
index 6abeb44a..8e9f6167 100644
--- a/src/hyperlight_host/src/hypervisor/kvm.rs
+++ b/src/hyperlight_host/src/hypervisor/kvm.rs
@@ -74,6 +74,7 @@ mod debug {
     /// KVM Debug struct
     /// This struct is used to abstract the internal details of the kvm
     /// guest debugging settings
+    #[derive(Default)]
     pub struct KvmDebug {
         /// vCPU stepping state
         single_step: bool,
@@ -167,6 +168,12 @@ mod debug {
     }
 
     impl KVMDriver {
+        /// Resets the debug information to disable debugging
+        fn disable_debug(&mut self) -> Result<()> {
+            self.debug = Some(KvmDebug::default());
+
+            self.set_single_step(false)
+        }
 
         /// Returns the instruction pointer from the stopped vCPU
         fn get_instruction_pointer(&self) -> Result<u64> {
@@ -342,6 +349,11 @@ mod debug {
                     self.set_single_step(false)?;
                     Ok(DebugResponse::Continue)
                 }
+                DebugMsg::DisableDebug => {
+                    self.disable_debug()?;
+
+                    Ok(DebugResponse::DisableDebug)
+                }
                 DebugMsg::ReadRegisters => {
                     let mut regs = X86_64Regs::default();
                     self.read_regs(&mut regs).expect("Read Regs error");
@@ -353,6 +365,10 @@ mod debug {
                         .expect("Remove hw breakpoint error");
                     Ok(DebugResponse::RemoveHwBreakpoint(res))
                 }
+                DebugMsg::Step => {
+                    self.set_single_step(true)?;
+                    Ok(DebugResponse::Step)
+                }
                 DebugMsg::WriteRegisters(regs) => {
                     self.write_regs(&regs).expect("Write Regs error");
                     Ok(DebugResponse::WriteRegisters)
@@ -720,7 +736,7 @@ impl Hypervisor for KVMDriver {
             // If the command was either step or continue, we need to run the vcpu
             let cont = matches!(
                 response,
-                DebugResponse::Continue
+                DebugResponse::Step | DebugResponse::Continue | DebugResponse::DisableDebug
             );
 
             self.send_dbg_msg(response)

From f2e13d70557a90b65df98c4c6e5a742929a96c77 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Doru=20Bl=C3=A2nzeanu?= <dblnz@pm.me>
Date: Tue, 4 Feb 2025 22:31:59 +0200
Subject: [PATCH 12/18] add a way to read/write host shared mem when debugging
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Signed-off-by: Doru Blânzeanu <dblnz@pm.me>
---
 .../src/hypervisor/handlers.rs                | 22 +++++++++
 .../src/hypervisor/hyperv_linux.rs            |  8 +++
 .../src/hypervisor/hyperv_windows.rs          |  8 +++
 .../src/hypervisor/hypervisor_handler.rs      | 10 ++++
 .../src/hypervisor/inprocess.rs               |  4 ++
 src/hyperlight_host/src/hypervisor/kvm.rs     | 49 +++++++++++++++++--
 src/hyperlight_host/src/hypervisor/mod.rs     | 13 ++++-
 src/hyperlight_host/src/mem/layout.rs         |  2 +-
 src/hyperlight_host/src/mem/shared_mem.rs     |  2 +-
 src/hyperlight_host/src/sandbox/mem_access.rs | 41 ++++++++++++++++
 .../src/sandbox/uninitialized.rs              |  1 +
 .../src/sandbox/uninitialized_evolve.rs       |  7 +++
 12 files changed, 161 insertions(+), 6 deletions(-)

diff --git a/src/hyperlight_host/src/hypervisor/handlers.rs b/src/hyperlight_host/src/hypervisor/handlers.rs
index b17fabfc..3ffd6444 100644
--- a/src/hyperlight_host/src/hypervisor/handlers.rs
+++ b/src/hyperlight_host/src/hypervisor/handlers.rs
@@ -102,3 +102,25 @@ impl MemAccessHandlerCaller for MemAccessHandler {
         func()
     }
 }
+
+/// The trait representing custom logic to handle the case when
+/// a Hypervisor's virtual CPU (vCPU) informs Hyperlight a debug memory access
+/// has been requested.
+#[cfg(gdb)]
+pub trait DbgMemAccessHandlerCaller: Send {
+    /// Function that gets called when a read is requested.
+    fn read(&mut self, addr: usize, data: &mut [u8]) -> Result<()>;
+
+    /// Function that gets called when a write is requested.
+    fn write(&mut self, addr: usize, data: &[u8]) -> Result<()>;
+
+    /// Function that gets called for a request to get guest code offset.
+    fn get_code_offset(&mut self) -> Result<usize>;
+}
+
+/// A convenient type representing an implementer of `DbgMemAccessHandlerCaller`
+///
+/// Note: This needs to be wrapped in a Mutex to be able to grab a mutable
+/// reference to the underlying data
+#[cfg(gdb)]
+pub type DbgMemAccessHandlerWrapper = Arc<Mutex<dyn DbgMemAccessHandlerCaller>>;
diff --git a/src/hyperlight_host/src/hypervisor/hyperv_linux.rs b/src/hyperlight_host/src/hypervisor/hyperv_linux.rs
index 33798d88..882b1c12 100644
--- a/src/hyperlight_host/src/hypervisor/hyperv_linux.rs
+++ b/src/hyperlight_host/src/hypervisor/hyperv_linux.rs
@@ -44,6 +44,8 @@ use mshv_ioctls::{Mshv, VcpuFd, VmFd};
 use tracing::{instrument, Span};
 
 use super::fpu::{FP_CONTROL_WORD_DEFAULT, FP_TAG_WORD_DEFAULT, MXCSR_DEFAULT};
+#[cfg(gdb)]
+use super::handlers::DbgMemAccessHandlerWrapper;
 use super::handlers::{MemAccessHandlerWrapper, OutBHandlerWrapper};
 use super::{
     Hypervisor, VirtualCPU, CR0_AM, CR0_ET, CR0_MP, CR0_NE, CR0_PE, CR0_PG, CR0_WP, CR4_OSFXSR,
@@ -203,6 +205,7 @@ impl Hypervisor for HypervLinuxDriver {
         outb_hdl: OutBHandlerWrapper,
         mem_access_hdl: MemAccessHandlerWrapper,
         hv_handler: Option<HypervisorHandler>,
+        #[cfg(gdb)] dbg_mem_access_fn: DbgMemAccessHandlerWrapper,
     ) -> Result<()> {
         let regs = StandardRegisters {
             rip: self.entrypoint,
@@ -224,6 +227,8 @@ impl Hypervisor for HypervLinuxDriver {
             hv_handler,
             outb_hdl,
             mem_access_hdl,
+            #[cfg(gdb)]
+            dbg_mem_access_fn,
         )?;
 
         // reset RSP to what it was before initialise
@@ -242,6 +247,7 @@ impl Hypervisor for HypervLinuxDriver {
         outb_handle_fn: OutBHandlerWrapper,
         mem_access_fn: MemAccessHandlerWrapper,
         hv_handler: Option<HypervisorHandler>,
+        #[cfg(gdb)] dbg_mem_access_fn: DbgMemAccessHandlerWrapper,
     ) -> Result<()> {
         // Reset general purpose registers except RSP, then set RIP
         let rsp_before = self.vcpu_fd.get_regs()?.rsp;
@@ -268,6 +274,8 @@ impl Hypervisor for HypervLinuxDriver {
             hv_handler,
             outb_handle_fn,
             mem_access_fn,
+            #[cfg(gdb)]
+            dbg_mem_access_fn,
         )?;
 
         // reset RSP to what it was before function call
diff --git a/src/hyperlight_host/src/hypervisor/hyperv_windows.rs b/src/hyperlight_host/src/hypervisor/hyperv_windows.rs
index 067d0cb1..acde5e8f 100644
--- a/src/hyperlight_host/src/hypervisor/hyperv_windows.rs
+++ b/src/hyperlight_host/src/hypervisor/hyperv_windows.rs
@@ -28,6 +28,8 @@ use windows::Win32::System::Hypervisor::{
 };
 
 use super::fpu::{FP_TAG_WORD_DEFAULT, MXCSR_DEFAULT};
+#[cfg(gdb)]
+use super::handlers::DbgMemAccessHandlerWrapper;
 use super::handlers::{MemAccessHandlerWrapper, OutBHandlerWrapper};
 use super::surrogate_process::SurrogateProcess;
 use super::surrogate_process_manager::*;
@@ -305,6 +307,7 @@ impl Hypervisor for HypervWindowsDriver {
         outb_hdl: OutBHandlerWrapper,
         mem_access_hdl: MemAccessHandlerWrapper,
         hv_handler: Option<HypervisorHandler>,
+        #[cfg(gdb)] dbg_mem_access_hdl: DbgMemAccessHandlerWrapper,
     ) -> Result<()> {
         let regs = WHvGeneralRegisters {
             rip: self.entrypoint,
@@ -326,6 +329,8 @@ impl Hypervisor for HypervWindowsDriver {
             hv_handler,
             outb_hdl,
             mem_access_hdl,
+            #[cfg(gdb)]
+            dbg_mem_access_hdl,
         )?;
 
         // reset RSP to what it was before initialise
@@ -344,6 +349,7 @@ impl Hypervisor for HypervWindowsDriver {
         outb_hdl: OutBHandlerWrapper,
         mem_access_hdl: MemAccessHandlerWrapper,
         hv_handler: Option<HypervisorHandler>,
+        #[cfg(gdb)] dbg_mem_access_hdl: DbgMemAccessHandlerWrapper,
     ) -> Result<()> {
         // Reset general purpose registers except RSP, then set RIP
         let rsp_before = self.processor.get_regs()?.rsp;
@@ -368,6 +374,8 @@ impl Hypervisor for HypervWindowsDriver {
             hv_handler,
             outb_hdl,
             mem_access_hdl,
+            #[cfg(gdb)]
+            dbg_mem_access_hdl,
         )?;
 
         // reset RSP to what it was before function call
diff --git a/src/hyperlight_host/src/hypervisor/hypervisor_handler.rs b/src/hyperlight_host/src/hypervisor/hypervisor_handler.rs
index cd1b8b91..af571874 100644
--- a/src/hyperlight_host/src/hypervisor/hypervisor_handler.rs
+++ b/src/hyperlight_host/src/hypervisor/hypervisor_handler.rs
@@ -37,6 +37,8 @@ use windows::Win32::System::Hypervisor::{WHvCancelRunVirtualProcessor, WHV_PARTI
 
 #[cfg(feature = "function_call_metrics")]
 use crate::histogram_vec_observe;
+#[cfg(gdb)]
+use crate::hypervisor::handlers::DbgMemAccessHandlerWrapper;
 use crate::hypervisor::handlers::{MemAccessHandlerWrapper, OutBHandlerWrapper};
 #[cfg(target_os = "windows")]
 use crate::hypervisor::wrappers::HandleWrapper;
@@ -187,6 +189,8 @@ pub(crate) struct HvHandlerConfig {
     pub(crate) outb_handler: OutBHandlerWrapper,
     pub(crate) mem_access_handler: MemAccessHandlerWrapper,
     pub(crate) max_wait_for_cancellation: Duration,
+    #[cfg(gdb)]
+    pub(crate) dbg_mem_access_handler: DbgMemAccessHandlerWrapper,
 }
 
 impl HypervisorHandler {
@@ -351,6 +355,8 @@ impl HypervisorHandler {
                                     configuration.outb_handler.clone(),
                                     configuration.mem_access_handler.clone(),
                                     Some(hv_handler_clone.clone()),
+                                    #[cfg(gdb)]
+                                    configuration.dbg_mem_access_handler.clone(),
                                 );
                                 drop(mem_lock_guard);
                                 drop(evar_lock_guard);
@@ -436,6 +442,8 @@ impl HypervisorHandler {
                                             configuration.outb_handler.clone(),
                                             configuration.mem_access_handler.clone(),
                                             Some(hv_handler_clone.clone()),
+                                            #[cfg(gdb)]
+                                            configuration.dbg_mem_access_handler.clone(),
                                         );
                                         histogram_vec_observe!(
                                             &GuestFunctionCallDurationMicroseconds,
@@ -451,6 +459,8 @@ impl HypervisorHandler {
                                         configuration.outb_handler.clone(),
                                         configuration.mem_access_handler.clone(),
                                         Some(hv_handler_clone.clone()),
+                                        #[cfg(gdb)]
+                                        configuration.dbg_mem_access_handler.clone(),
                                     )
                                 };
                                 drop(mem_lock_guard);
diff --git a/src/hyperlight_host/src/hypervisor/inprocess.rs b/src/hyperlight_host/src/hypervisor/inprocess.rs
index f7076a5b..1fec3df5 100644
--- a/src/hyperlight_host/src/hypervisor/inprocess.rs
+++ b/src/hyperlight_host/src/hypervisor/inprocess.rs
@@ -17,6 +17,8 @@ limitations under the License.
 use std::fmt::Debug;
 use std::os::raw::c_void;
 
+#[cfg(gdb)]
+use super::handlers::DbgMemAccessHandlerWrapper;
 use super::{HyperlightExit, Hypervisor};
 #[cfg(crashdump)]
 use crate::mem::memory_region::MemoryRegion;
@@ -73,6 +75,7 @@ impl<'a> Hypervisor for InprocessDriver<'a> {
         _outb_handle_fn: super::handlers::OutBHandlerWrapper,
         _mem_access_fn: super::handlers::MemAccessHandlerWrapper,
         _hv_handler: Option<super::hypervisor_handler::HypervisorHandler>,
+        #[cfg(gdb)] _dbg_mem_access_fn: DbgMemAccessHandlerWrapper,
     ) -> crate::Result<()> {
         let entrypoint_fn: extern "win64" fn(u64, u64, u64, u64) =
             unsafe { std::mem::transmute(self.args.entrypoint_raw as *const c_void) };
@@ -93,6 +96,7 @@ impl<'a> Hypervisor for InprocessDriver<'a> {
         _outb_handle_fn: super::handlers::OutBHandlerWrapper,
         _mem_access_fn: super::handlers::MemAccessHandlerWrapper,
         _hv_handler: Option<super::hypervisor_handler::HypervisorHandler>,
+        #[cfg(gdb)] _dbg_mem_access_fn: DbgMemAccessHandlerWrapper,
     ) -> crate::Result<()> {
         let ptr: u64 = dispatch_func_addr.into();
         let dispatch_func: extern "win64" fn() =
diff --git a/src/hyperlight_host/src/hypervisor/kvm.rs b/src/hyperlight_host/src/hypervisor/kvm.rs
index 8e9f6167..81c525d1 100644
--- a/src/hyperlight_host/src/hypervisor/kvm.rs
+++ b/src/hyperlight_host/src/hypervisor/kvm.rs
@@ -16,6 +16,8 @@ limitations under the License.
 
 use std::convert::TryFrom;
 use std::fmt::Debug;
+#[cfg(gdb)]
+use std::sync::{Arc, Mutex};
 
 use kvm_bindings::{kvm_fpu, kvm_regs, kvm_userspace_memory_region, KVM_MEM_READONLY};
 use kvm_ioctls::Cap::UserMemory;
@@ -24,7 +26,9 @@ use tracing::{instrument, Span};
 
 use super::fpu::{FP_CONTROL_WORD_DEFAULT, FP_TAG_WORD_DEFAULT, MXCSR_DEFAULT};
 #[cfg(gdb)]
-use super::gdb::{create_gdb_thread, DebugMsg, DebugResponse, DebugCommChannel, VcpuStopReason};
+use super::gdb::{create_gdb_thread, DebugCommChannel, DebugMsg, DebugResponse, VcpuStopReason};
+#[cfg(gdb)]
+use super::handlers::DbgMemAccessHandlerWrapper;
 use super::handlers::{MemAccessHandlerWrapper, OutBHandlerWrapper};
 use super::{
     HyperlightExit, Hypervisor, VirtualCPU, CR0_AM, CR0_ET, CR0_MP, CR0_NE, CR0_PE, CR0_PG, CR0_WP,
@@ -61,6 +65,7 @@ pub(crate) fn is_hypervisor_present() -> bool {
 
 #[cfg(gdb)]
 mod debug {
+    use std::sync::{Arc, Mutex};
     use kvm_bindings::{
         kvm_guest_debug, kvm_regs, KVM_GUESTDBG_ENABLE, KVM_GUESTDBG_SINGLESTEP,
         KVM_GUESTDBG_USE_HW_BP, KVM_GUESTDBG_USE_SW_BP,
@@ -69,6 +74,7 @@ mod debug {
 
     use super::KVMDriver;
     use crate::hypervisor::gdb::{DebugMsg, DebugResponse, VcpuStopReason, X86_64Regs};
+    use crate::hypervisor::handlers::DbgMemAccessHandlerCaller;
     use crate::{new_error, Result};
 
     /// KVM Debug struct
@@ -337,6 +343,7 @@ mod debug {
         pub fn process_dbg_request(
             &mut self,
             req: DebugMsg,
+            _dbg_mem_access_fn: Arc<Mutex<dyn DbgMemAccessHandlerCaller>>,
         ) -> Result<DebugResponse> {
             match req {
                 DebugMsg::AddHwBreakpoint(addr) => {
@@ -548,6 +555,7 @@ impl Hypervisor for KVMDriver {
         outb_hdl: OutBHandlerWrapper,
         mem_access_hdl: MemAccessHandlerWrapper,
         hv_handler: Option<HypervisorHandler>,
+        #[cfg(gdb)] dbg_mem_access_fn: DbgMemAccessHandlerWrapper,
     ) -> Result<()> {
         let regs = kvm_regs {
             rip: self.entrypoint,
@@ -568,6 +576,8 @@ impl Hypervisor for KVMDriver {
             hv_handler,
             outb_hdl,
             mem_access_hdl,
+            #[cfg(gdb)]
+            dbg_mem_access_fn,
         )?;
 
         // reset RSP to what it was before initialise
@@ -585,6 +595,7 @@ impl Hypervisor for KVMDriver {
         outb_handle_fn: OutBHandlerWrapper,
         mem_access_fn: MemAccessHandlerWrapper,
         hv_handler: Option<HypervisorHandler>,
+        #[cfg(gdb)] dbg_mem_access_fn: DbgMemAccessHandlerWrapper,
     ) -> Result<()> {
         // Reset general purpose registers except RSP, then set RIP
         let rsp_before = self.vcpu_fd.get_regs()?.rsp;
@@ -610,6 +621,8 @@ impl Hypervisor for KVMDriver {
             hv_handler,
             outb_handle_fn,
             mem_access_fn,
+            #[cfg(gdb)]
+            dbg_mem_access_fn,
         )?;
 
         // reset RSP to what it was before function call
@@ -721,6 +734,7 @@ impl Hypervisor for KVMDriver {
     #[cfg(gdb)]
     fn handle_debug(
         &mut self,
+        dbg_mem_access_fn: Arc<Mutex<dyn super::handlers::DbgMemAccessHandlerCaller>>,
         stop_reason: VcpuStopReason,
     ) -> Result<()> {
         self.send_dbg_msg(DebugResponse::VcpuStopped(stop_reason))
@@ -731,7 +745,7 @@ impl Hypervisor for KVMDriver {
             // Wait for a message from gdb
             let req = self.recv_dbg_msg()?;
 
-            let response = self.process_dbg_request(req)?;
+            let response = self.process_dbg_request(req, dbg_mem_access_fn.clone())?;
 
             // If the command was either step or continue, we need to run the vcpu
             let cont = matches!(
@@ -755,10 +769,30 @@ impl Hypervisor for KVMDriver {
 mod tests {
     use std::sync::{Arc, Mutex};
 
+    #[cfg(gdb)]
+    use crate::hypervisor::handlers::DbgMemAccessHandlerCaller;
     use crate::hypervisor::handlers::{MemAccessHandler, OutBHandler};
     use crate::hypervisor::tests::test_initialise;
     use crate::Result;
 
+    #[cfg(gdb)]
+    struct DbgMemAccessHandler {}
+
+    #[cfg(gdb)]
+    impl DbgMemAccessHandlerCaller for DbgMemAccessHandler {
+        fn read(&mut self, _offset: usize, _data: &mut [u8]) -> Result<()> {
+            Ok(())
+        }
+
+        fn write(&mut self, _offset: usize, _data: &[u8]) -> Result<()> {
+            Ok(())
+        }
+
+        fn get_code_offset(&mut self) -> Result<usize> {
+            Ok(0)
+        }
+    }
+
     #[test]
     fn test_init() {
         if !super::is_hypervisor_present() {
@@ -774,6 +808,15 @@ mod tests {
             let func: Box<dyn FnMut() -> Result<()> + Send> = Box::new(|| -> Result<()> { Ok(()) });
             Arc::new(Mutex::new(MemAccessHandler::from(func)))
         };
-        test_initialise(outb_handler, mem_access_handler).unwrap();
+        #[cfg(gdb)]
+        let dbg_mem_access_handler = Arc::new(Mutex::new(DbgMemAccessHandler {}));
+
+        test_initialise(
+            outb_handler,
+            mem_access_handler,
+            #[cfg(gdb)]
+            dbg_mem_access_handler,
+        )
+        .unwrap();
     }
 }
diff --git a/src/hyperlight_host/src/hypervisor/mod.rs b/src/hyperlight_host/src/hypervisor/mod.rs
index 6ca78664..9a24679d 100644
--- a/src/hyperlight_host/src/hypervisor/mod.rs
+++ b/src/hyperlight_host/src/hypervisor/mod.rs
@@ -68,6 +68,8 @@ use std::sync::{Arc, Mutex};
 #[cfg(gdb)]
 use gdb::VcpuStopReason;
 
+#[cfg(gdb)]
+use self::handlers::{DbgMemAccessHandlerCaller, DbgMemAccessHandlerWrapper};
 use self::handlers::{
     MemAccessHandlerCaller, MemAccessHandlerWrapper, OutBHandlerCaller, OutBHandlerWrapper,
 };
@@ -128,6 +130,7 @@ pub(crate) trait Hypervisor: Debug + Sync + Send {
         outb_handle_fn: OutBHandlerWrapper,
         mem_access_fn: MemAccessHandlerWrapper,
         hv_handler: Option<HypervisorHandler>,
+        #[cfg(gdb)] dbg_mem_access_fn: DbgMemAccessHandlerWrapper,
     ) -> Result<()>;
 
     /// Dispatch a call from the host to the guest using the given pointer
@@ -143,6 +146,7 @@ pub(crate) trait Hypervisor: Debug + Sync + Send {
         outb_handle_fn: OutBHandlerWrapper,
         mem_access_fn: MemAccessHandlerWrapper,
         hv_handler: Option<HypervisorHandler>,
+        #[cfg(gdb)] dbg_mem_access_fn: DbgMemAccessHandlerWrapper,
     ) -> Result<()>;
 
     /// Handle an IO exit from the internally stored vCPU.
@@ -204,6 +208,7 @@ pub(crate) trait Hypervisor: Debug + Sync + Send {
     /// handles the cases when the vCPU stops due to a Debug event
     fn handle_debug(
         &mut self,
+        _dbg_mem_access_fn: Arc<Mutex<dyn DbgMemAccessHandlerCaller>>,
         _stop_reason: VcpuStopReason,
     ) -> Result<()> {
         unimplemented!()
@@ -221,12 +226,13 @@ impl VirtualCPU {
         hv_handler: Option<HypervisorHandler>,
         outb_handle_fn: Arc<Mutex<dyn OutBHandlerCaller>>,
         mem_access_fn: Arc<Mutex<dyn MemAccessHandlerCaller>>,
+        #[cfg(gdb)] dbg_mem_access_fn: Arc<Mutex<dyn DbgMemAccessHandlerCaller>>,
     ) -> Result<()> {
         loop {
             match hv.run() {
                 #[cfg(gdb)]
                 Ok(HyperlightExit::Debug(stop_reason)) => {
-                    hv.handle_debug(stop_reason)?;
+                    hv.handle_debug(dbg_mem_access_fn.clone(), stop_reason)?;
                 }
 
                 Ok(HyperlightExit::Halt()) => {
@@ -301,6 +307,8 @@ pub(crate) mod tests {
 
     use hyperlight_testing::dummy_guest_as_string;
 
+    #[cfg(gdb)]
+    use super::handlers::DbgMemAccessHandlerWrapper;
     use super::handlers::{MemAccessHandlerWrapper, OutBHandlerWrapper};
     use crate::hypervisor::hypervisor_handler::{
         HvHandlerConfig, HypervisorHandler, HypervisorHandlerAction,
@@ -313,6 +321,7 @@ pub(crate) mod tests {
     pub(crate) fn test_initialise(
         outb_hdl: OutBHandlerWrapper,
         mem_access_hdl: MemAccessHandlerWrapper,
+        #[cfg(gdb)] dbg_mem_access_fn: DbgMemAccessHandlerWrapper,
     ) -> Result<()> {
         let filename = dummy_guest_as_string().map_err(|e| new_error!("{}", e))?;
         if !Path::new(&filename).exists() {
@@ -330,6 +339,8 @@ pub(crate) mod tests {
         let hv_handler_config = HvHandlerConfig {
             outb_handler: outb_hdl,
             mem_access_handler: mem_access_hdl,
+            #[cfg(gdb)]
+            dbg_mem_access_handler: dbg_mem_access_fn,
             seed: 1234567890,
             page_size: 4096,
             peb_addr: RawPtr::from(0x230000),
diff --git a/src/hyperlight_host/src/mem/layout.rs b/src/hyperlight_host/src/mem/layout.rs
index 251032a2..3a13879c 100644
--- a/src/hyperlight_host/src/mem/layout.rs
+++ b/src/hyperlight_host/src/mem/layout.rs
@@ -656,7 +656,7 @@ impl SandboxMemoryLayout {
 
     /// Get the guest address of the code section in the sandbox
     #[instrument(skip_all, parent = Span::current(), level= "Trace")]
-    pub(super) fn get_guest_code_address(&self) -> usize {
+    pub(crate) fn get_guest_code_address(&self) -> usize {
         Self::BASE_ADDRESS + self.guest_code_offset
     }
 
diff --git a/src/hyperlight_host/src/mem/shared_mem.rs b/src/hyperlight_host/src/mem/shared_mem.rs
index 0733f85d..721704b4 100644
--- a/src/hyperlight_host/src/mem/shared_mem.rs
+++ b/src/hyperlight_host/src/mem/shared_mem.rs
@@ -840,7 +840,7 @@ impl HostSharedMemory {
         Ok(())
     }
 
-    /// /Copy the contents of the sandbox at the specified offset into
+    /// Copy the contents of the sandbox at the specified offset into
     /// the slice
     pub fn copy_from_slice(&self, slice: &[u8], offset: usize) -> Result<()> {
         bounds_check!(offset, slice.len(), self.mem_size());
diff --git a/src/hyperlight_host/src/sandbox/mem_access.rs b/src/hyperlight_host/src/sandbox/mem_access.rs
index 6a1e6b70..7bf4fbdc 100644
--- a/src/hyperlight_host/src/sandbox/mem_access.rs
+++ b/src/hyperlight_host/src/sandbox/mem_access.rs
@@ -20,6 +20,8 @@ use tracing::{instrument, Span};
 
 use super::mem_mgr::MemMgrWrapper;
 use crate::error::HyperlightError::StackOverflow;
+#[cfg(gdb)]
+use crate::hypervisor::handlers::{DbgMemAccessHandlerCaller, DbgMemAccessHandlerWrapper};
 use crate::hypervisor::handlers::{
     MemAccessHandler, MemAccessHandlerFunction, MemAccessHandlerWrapper,
 };
@@ -44,3 +46,42 @@ pub(crate) fn mem_access_handler_wrapper(
     let mem_access_hdl = MemAccessHandler::from(mem_access_func);
     Arc::new(Mutex::new(mem_access_hdl))
 }
+
+#[cfg(gdb)]
+struct DbgMemAccessContainer {
+    wrapper: MemMgrWrapper<HostSharedMemory>,
+}
+
+#[cfg(gdb)]
+impl DbgMemAccessHandlerCaller for DbgMemAccessContainer {
+    #[instrument(err(Debug), skip_all, parent = Span::current(), level= "Trace")]
+    fn read(&mut self, addr: usize, data: &mut [u8]) -> Result<()> {
+        self.wrapper
+            .unwrap_mgr_mut()
+            .get_shared_mem_mut()
+            .copy_to_slice(data, addr)
+    }
+
+    #[instrument(err(Debug), skip_all, parent = Span::current(), level= "Trace")]
+    fn write(&mut self, addr: usize, data: &[u8]) -> Result<()> {
+        self.wrapper
+            .unwrap_mgr_mut()
+            .get_shared_mem_mut()
+            .copy_from_slice(data, addr)
+    }
+
+    #[instrument(err(Debug), skip_all, parent = Span::current(), level= "Trace")]
+    fn get_code_offset(&mut self) -> Result<usize> {
+        Ok(self.wrapper.unwrap_mgr().layout.get_guest_code_address())
+    }
+}
+
+#[cfg(gdb)]
+#[instrument(skip_all, parent = Span::current(), level= "Trace")]
+pub(crate) fn dbg_mem_access_handler_wrapper(
+    wrapper: MemMgrWrapper<HostSharedMemory>,
+) -> DbgMemAccessHandlerWrapper {
+    let container = DbgMemAccessContainer { wrapper };
+
+    Arc::new(Mutex::new(container))
+}
diff --git a/src/hyperlight_host/src/sandbox/uninitialized.rs b/src/hyperlight_host/src/sandbox/uninitialized.rs
index e3f8e0bd..e38647d8 100644
--- a/src/hyperlight_host/src/sandbox/uninitialized.rs
+++ b/src/hyperlight_host/src/sandbox/uninitialized.rs
@@ -165,6 +165,7 @@ impl UninitializedSandbox {
         }
 
         let sandbox_cfg = cfg.unwrap_or_default();
+
         #[cfg(gdb)]
         let debug_info = sandbox_cfg.get_guest_debug_info();
         let mut mem_mgr_wrapper = {
diff --git a/src/hyperlight_host/src/sandbox/uninitialized_evolve.rs b/src/hyperlight_host/src/sandbox/uninitialized_evolve.rs
index 55252c92..f9e99d7a 100644
--- a/src/hyperlight_host/src/sandbox/uninitialized_evolve.rs
+++ b/src/hyperlight_host/src/sandbox/uninitialized_evolve.rs
@@ -20,6 +20,8 @@ use std::sync::{Arc, Mutex};
 use rand::Rng;
 use tracing::{instrument, Span};
 
+#[cfg(gdb)]
+use super::mem_access::dbg_mem_access_handler_wrapper;
 use crate::hypervisor::hypervisor_handler::{
     HvHandlerConfig, HypervisorHandler, HypervisorHandlerAction,
 };
@@ -106,6 +108,9 @@ fn hv_init(
 ) -> Result<HypervisorHandler> {
     let outb_hdl = outb_handler_wrapper(hshm.clone(), host_funcs);
     let mem_access_hdl = mem_access_handler_wrapper(hshm.clone());
+    #[cfg(gdb)]
+    let dbg_mem_access_hdl = dbg_mem_access_handler_wrapper(hshm.clone());
+
     let seed = {
         let mut rng = rand::rng();
         rng.random::<u64>()
@@ -118,6 +123,8 @@ fn hv_init(
     let hv_handler_config = HvHandlerConfig {
         outb_handler: outb_hdl,
         mem_access_handler: mem_access_hdl,
+        #[cfg(gdb)]
+        dbg_mem_access_handler: dbg_mem_access_hdl,
         seed,
         page_size,
         peb_addr,

From d3da3b11b1f8f2941e5425ab66098e0c350f997e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Doru=20Bl=C3=A2nzeanu?= <dblnz@pm.me>
Date: Tue, 4 Feb 2025 22:36:38 +0200
Subject: [PATCH 13/18] add read/write address and text section offset support
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Signed-off-by: Doru Blânzeanu <dblnz@pm.me>
---
 .../src/hypervisor/gdb/event_loop.rs          |  2 +-
 src/hyperlight_host/src/hypervisor/gdb/mod.rs |  6 ++
 .../src/hypervisor/gdb/x86_64_target.rs       | 46 +++++++++-
 src/hyperlight_host/src/hypervisor/kvm.rs     | 85 ++++++++++++++++++-
 4 files changed, 134 insertions(+), 5 deletions(-)

diff --git a/src/hyperlight_host/src/hypervisor/gdb/event_loop.rs b/src/hyperlight_host/src/hypervisor/gdb/event_loop.rs
index af46c3ae..9b456ff7 100644
--- a/src/hyperlight_host/src/hypervisor/gdb/event_loop.rs
+++ b/src/hyperlight_host/src/hypervisor/gdb/event_loop.rs
@@ -99,7 +99,7 @@ pub fn event_loop_thread(
 ) {
     match debugger.run_blocking::<GdbBlockingEventLoop>(target) {
         Ok(disconnect_reason) => match disconnect_reason {
-            DisconnectReason::Disconnect => { 
+            DisconnectReason::Disconnect => {
                 log::info!("Gdb client disconnected");
                 if let Err(e) = target.disable_debug() {
                     log::error!("Cannot disable debugging: {:?}", e);
diff --git a/src/hyperlight_host/src/hypervisor/gdb/mod.rs b/src/hyperlight_host/src/hypervisor/gdb/mod.rs
index 4570f08f..47a7922c 100644
--- a/src/hyperlight_host/src/hypervisor/gdb/mod.rs
+++ b/src/hyperlight_host/src/hypervisor/gdb/mod.rs
@@ -102,9 +102,12 @@ pub enum DebugMsg {
     AddHwBreakpoint(u64),
     Continue,
     DisableDebug,
+    GetCodeSectionOffset,
+    ReadAddr(u64, usize),
     ReadRegisters,
     RemoveHwBreakpoint(u64),
     Step,
+    WriteAddr(u64, Vec<u8>),
     WriteRegisters(X86_64Regs),
 }
 
@@ -114,10 +117,13 @@ pub enum DebugResponse {
     AddHwBreakpoint(bool),
     Continue,
     DisableDebug,
+    GetCodeSectionOffset(u64),
+    ReadAddr(Vec<u8>),
     ReadRegisters(X86_64Regs),
     RemoveHwBreakpoint(bool),
     Step,
     VcpuStopped(VcpuStopReason),
+    WriteAddr,
     WriteRegisters,
 }
 
diff --git a/src/hyperlight_host/src/hypervisor/gdb/x86_64_target.rs b/src/hyperlight_host/src/hypervisor/gdb/x86_64_target.rs
index 70fd43ab..0384f844 100644
--- a/src/hyperlight_host/src/hypervisor/gdb/x86_64_target.rs
+++ b/src/hyperlight_host/src/hypervisor/gdb/x86_64_target.rs
@@ -25,10 +25,11 @@ use gdbstub::target::ext::base::BaseOps;
 use gdbstub::target::ext::breakpoints::{
     Breakpoints, BreakpointsOps, HwBreakpoint, HwBreakpointOps,
 };
+use gdbstub::target::ext::section_offsets::{Offsets, SectionOffsets};
 use gdbstub::target::{Target, TargetError, TargetResult};
 use gdbstub_arch::x86::X86_64_SSE as GdbTargetArch;
 
-use super::{DebugMsg, DebugResponse, DebugCommChannel, GdbTargetError, X86_64Regs};
+use super::{DebugCommChannel, DebugMsg, DebugResponse, GdbTargetError, X86_64Regs};
 
 /// Gdbstub target used by the gdbstub crate to provide GDB protocol implementation
 pub struct HyperlightSandboxTarget {
@@ -106,6 +107,12 @@ impl Target for HyperlightSandboxTarget {
     fn base_ops(&mut self) -> BaseOps<Self::Arch, Self::Error> {
         BaseOps::SingleThread(self)
     }
+
+    fn support_section_offsets(
+        &mut self,
+    ) -> Option<gdbstub::target::ext::section_offsets::SectionOffsetsOps<Self>> {
+        Some(self)
+    }
 }
 
 impl SingleThreadBase for HyperlightSandboxTarget {
@@ -116,7 +123,17 @@ impl SingleThreadBase for HyperlightSandboxTarget {
     ) -> TargetResult<usize, Self> {
         log::debug!("Read addr: {:X} len: {:X}", gva, data.len());
 
-        unimplemented!()
+        match self.send_command(DebugMsg::ReadAddr(gva, data.len()))? {
+            DebugResponse::ReadAddr(v) => {
+                data.copy_from_slice(&v);
+
+                Ok(v.len())
+            }
+            msg => {
+                log::error!("Unexpected message received: {:?}", msg);
+                Err(TargetError::Fatal(GdbTargetError::UnexpectedMessage))
+            }
+        }
     }
 
     fn write_addrs(
@@ -125,8 +142,15 @@ impl SingleThreadBase for HyperlightSandboxTarget {
         data: &[u8],
     ) -> TargetResult<(), Self> {
         log::debug!("Write addr: {:X} len: {:X}", gva, data.len());
+        let v = Vec::from(data);
 
-        unimplemented!()
+        match self.send_command(DebugMsg::WriteAddr(gva, v))? {
+            DebugResponse::WriteAddr => Ok(()),
+            msg => {
+                log::error!("Unexpected message received: {:?}", msg);
+                Err(TargetError::Fatal(GdbTargetError::UnexpectedMessage))
+            }
+        }
     }
 
     fn read_registers(
@@ -206,6 +230,22 @@ impl SingleThreadBase for HyperlightSandboxTarget {
         Some(self)
     }
 }
+impl SectionOffsets for HyperlightSandboxTarget {
+    fn get_section_offsets(&mut self) -> Result<Offsets<<Self::Arch as Arch>::Usize>, Self::Error> {
+        log::debug!("Get section offsets");
+
+        match self.send_command(DebugMsg::GetCodeSectionOffset)? {
+            DebugResponse::GetCodeSectionOffset(text) => Ok(Offsets::Segments {
+                text_seg: text,
+                data_seg: None,
+            }),
+            msg => {
+                log::error!("Unexpected message received: {:?}", msg);
+                Err(GdbTargetError::UnexpectedMessage)
+            }
+        }
+    }
+}
 
 impl Breakpoints for HyperlightSandboxTarget {
     fn support_hw_breakpoint(&mut self) -> Option<HwBreakpointOps<Self>> {
diff --git a/src/hyperlight_host/src/hypervisor/kvm.rs b/src/hyperlight_host/src/hypervisor/kvm.rs
index 81c525d1..b9324334 100644
--- a/src/hyperlight_host/src/hypervisor/kvm.rs
+++ b/src/hyperlight_host/src/hypervisor/kvm.rs
@@ -66,6 +66,8 @@ pub(crate) fn is_hypervisor_present() -> bool {
 #[cfg(gdb)]
 mod debug {
     use std::sync::{Arc, Mutex};
+
+    use hyperlight_common::mem::PAGE_SIZE;
     use kvm_bindings::{
         kvm_guest_debug, kvm_regs, KVM_GUESTDBG_ENABLE, KVM_GUESTDBG_SINGLESTEP,
         KVM_GUESTDBG_USE_HW_BP, KVM_GUESTDBG_USE_SW_BP,
@@ -75,6 +77,7 @@ mod debug {
     use super::KVMDriver;
     use crate::hypervisor::gdb::{DebugMsg, DebugResponse, VcpuStopReason, X86_64Regs};
     use crate::hypervisor::handlers::DbgMemAccessHandlerCaller;
+    use crate::mem::layout::SandboxMemoryLayout;
     use crate::{new_error, Result};
 
     /// KVM Debug struct
@@ -221,6 +224,66 @@ mod debug {
             }
         }
 
+        fn read_addrs(
+            &mut self,
+            mut gva: u64,
+            mut data: &mut [u8],
+            dbg_mem_access_fn: Arc<Mutex<dyn DbgMemAccessHandlerCaller>>,
+        ) -> Result<()> {
+            let data_len = data.len();
+            log::debug!("Read addr: {:X} len: {:X}", gva, data_len);
+
+            while !data.is_empty() {
+                let gpa = self.translate_gva(gva)?;
+
+                let read_len = std::cmp::min(
+                    data.len(),
+                    (PAGE_SIZE - (gpa & (PAGE_SIZE - 1))).try_into().unwrap(),
+                );
+                let offset = gpa as usize - SandboxMemoryLayout::BASE_ADDRESS;
+
+                dbg_mem_access_fn
+                    .try_lock()
+                    .map_err(|e| new_error!("Error locking at {}:{}: {}", file!(), line!(), e))?
+                    .read(offset, &mut data[..read_len])?;
+
+                data = &mut data[read_len..];
+                gva += read_len as u64;
+            }
+
+            Ok(())
+        }
+
+        fn write_addrs(
+            &mut self,
+            mut gva: u64,
+            mut data: &[u8],
+            dbg_mem_access_fn: Arc<Mutex<dyn DbgMemAccessHandlerCaller>>,
+        ) -> Result<()> {
+            let data_len = data.len();
+            log::debug!("Write addr: {:X} len: {:X}", gva, data_len);
+
+            while !data.is_empty() {
+                let gpa = self.translate_gva(gva)?;
+
+                let write_len = std::cmp::min(
+                    data.len(),
+                    (PAGE_SIZE - (gpa & (PAGE_SIZE - 1))).try_into().unwrap(),
+                );
+                let offset = gpa as usize - SandboxMemoryLayout::BASE_ADDRESS;
+
+                dbg_mem_access_fn
+                    .try_lock()
+                    .map_err(|e| new_error!("Error locking at {}:{}: {}", file!(), line!(), e))?
+                    .write(offset, data)?;
+
+                data = &data[write_len..];
+                gva += write_len as u64;
+            }
+
+            Ok(())
+        }
+
         fn read_regs(&self, regs: &mut X86_64Regs) -> Result<()> {
             log::debug!("Read registers");
             let vcpu_regs = self
@@ -343,7 +406,7 @@ mod debug {
         pub fn process_dbg_request(
             &mut self,
             req: DebugMsg,
-            _dbg_mem_access_fn: Arc<Mutex<dyn DbgMemAccessHandlerCaller>>,
+            dbg_mem_access_fn: Arc<Mutex<dyn DbgMemAccessHandlerCaller>>,
         ) -> Result<DebugResponse> {
             match req {
                 DebugMsg::AddHwBreakpoint(addr) => {
@@ -361,6 +424,21 @@ mod debug {
 
                     Ok(DebugResponse::DisableDebug)
                 }
+                DebugMsg::GetCodeSectionOffset => {
+                    let offset = dbg_mem_access_fn
+                        .try_lock()
+                        .map_err(|e| new_error!("Error locking at {}:{}: {}", file!(), line!(), e))?
+                        .get_code_offset()?;
+
+                    Ok(DebugResponse::GetCodeSectionOffset(offset as u64))
+                }
+                DebugMsg::ReadAddr(addr, len) => {
+                    let mut data = vec![0u8; len];
+
+                    self.read_addrs(addr, &mut data, dbg_mem_access_fn)?;
+
+                    Ok(DebugResponse::ReadAddr(data))
+                }
                 DebugMsg::ReadRegisters => {
                     let mut regs = X86_64Regs::default();
                     self.read_regs(&mut regs).expect("Read Regs error");
@@ -376,6 +454,11 @@ mod debug {
                     self.set_single_step(true)?;
                     Ok(DebugResponse::Step)
                 }
+                DebugMsg::WriteAddr(addr, data) => {
+                    self.write_addrs(addr, &data, dbg_mem_access_fn)?;
+
+                    Ok(DebugResponse::WriteAddr)
+                }
                 DebugMsg::WriteRegisters(regs) => {
                     self.write_regs(&regs).expect("Write Regs error");
                     Ok(DebugResponse::WriteRegisters)

From 04bd6b37db38e59dd529f260f6d676ca112bcad3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Doru=20Bl=C3=A2nzeanu?= <dblnz@pm.me>
Date: Tue, 4 Feb 2025 22:39:38 +0200
Subject: [PATCH 14/18] add sw breakpoint support
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Signed-off-by: Doru Blânzeanu <dblnz@pm.me>
---
 src/hyperlight_host/src/hypervisor/gdb/mod.rs |  4 +
 .../src/hypervisor/gdb/x86_64_target.rs       | 40 +++++++-
 src/hyperlight_host/src/hypervisor/kvm.rs     | 91 +++++++++++++++++++
 3 files changed, 134 insertions(+), 1 deletion(-)

diff --git a/src/hyperlight_host/src/hypervisor/gdb/mod.rs b/src/hyperlight_host/src/hypervisor/gdb/mod.rs
index 47a7922c..b18186a8 100644
--- a/src/hyperlight_host/src/hypervisor/gdb/mod.rs
+++ b/src/hyperlight_host/src/hypervisor/gdb/mod.rs
@@ -100,12 +100,14 @@ pub enum VcpuStopReason {
 #[derive(Debug)]
 pub enum DebugMsg {
     AddHwBreakpoint(u64),
+    AddSwBreakpoint(u64),
     Continue,
     DisableDebug,
     GetCodeSectionOffset,
     ReadAddr(u64, usize),
     ReadRegisters,
     RemoveHwBreakpoint(u64),
+    RemoveSwBreakpoint(u64),
     Step,
     WriteAddr(u64, Vec<u8>),
     WriteRegisters(X86_64Regs),
@@ -115,12 +117,14 @@ pub enum DebugMsg {
 #[derive(Debug)]
 pub enum DebugResponse {
     AddHwBreakpoint(bool),
+    AddSwBreakpoint(bool),
     Continue,
     DisableDebug,
     GetCodeSectionOffset(u64),
     ReadAddr(Vec<u8>),
     ReadRegisters(X86_64Regs),
     RemoveHwBreakpoint(bool),
+    RemoveSwBreakpoint(bool),
     Step,
     VcpuStopped(VcpuStopReason),
     WriteAddr,
diff --git a/src/hyperlight_host/src/hypervisor/gdb/x86_64_target.rs b/src/hyperlight_host/src/hypervisor/gdb/x86_64_target.rs
index 0384f844..470e5993 100644
--- a/src/hyperlight_host/src/hypervisor/gdb/x86_64_target.rs
+++ b/src/hyperlight_host/src/hypervisor/gdb/x86_64_target.rs
@@ -23,7 +23,7 @@ use gdbstub::target::ext::base::singlethread::{
 };
 use gdbstub::target::ext::base::BaseOps;
 use gdbstub::target::ext::breakpoints::{
-    Breakpoints, BreakpointsOps, HwBreakpoint, HwBreakpointOps,
+    Breakpoints, BreakpointsOps, HwBreakpoint, HwBreakpointOps, SwBreakpoint, SwBreakpointOps,
 };
 use gdbstub::target::ext::section_offsets::{Offsets, SectionOffsets};
 use gdbstub::target::{Target, TargetError, TargetResult};
@@ -230,6 +230,7 @@ impl SingleThreadBase for HyperlightSandboxTarget {
         Some(self)
     }
 }
+
 impl SectionOffsets for HyperlightSandboxTarget {
     fn get_section_offsets(&mut self) -> Result<Offsets<<Self::Arch as Arch>::Usize>, Self::Error> {
         log::debug!("Get section offsets");
@@ -251,6 +252,9 @@ impl Breakpoints for HyperlightSandboxTarget {
     fn support_hw_breakpoint(&mut self) -> Option<HwBreakpointOps<Self>> {
         Some(self)
     }
+    fn support_sw_breakpoint(&mut self) -> Option<SwBreakpointOps<'_, Self>> {
+        Some(self)
+    }
 }
 
 impl HwBreakpoint for HyperlightSandboxTarget {
@@ -287,6 +291,40 @@ impl HwBreakpoint for HyperlightSandboxTarget {
     }
 }
 
+impl SwBreakpoint for HyperlightSandboxTarget {
+    fn add_sw_breakpoint(
+        &mut self,
+        addr: <Self::Arch as Arch>::Usize,
+        _kind: <Self::Arch as Arch>::BreakpointKind,
+    ) -> TargetResult<bool, Self> {
+        log::debug!("Add sw breakpoint at address {:X}", addr);
+
+        match self.send_command(DebugMsg::AddSwBreakpoint(addr))? {
+            DebugResponse::AddSwBreakpoint(rsp) => Ok(rsp),
+            msg => {
+                log::error!("Unexpected message received: {:?}", msg);
+                Err(TargetError::Fatal(GdbTargetError::UnexpectedMessage))
+            }
+        }
+    }
+
+    fn remove_sw_breakpoint(
+        &mut self,
+        addr: <Self::Arch as Arch>::Usize,
+        _kind: <Self::Arch as Arch>::BreakpointKind,
+    ) -> TargetResult<bool, Self> {
+        log::debug!("Remove sw breakpoint at address {:X}", addr);
+
+        match self.send_command(DebugMsg::RemoveSwBreakpoint(addr))? {
+            DebugResponse::RemoveSwBreakpoint(rsp) => Ok(rsp),
+            msg => {
+                log::error!("Unexpected message received: {:?}", msg);
+                Err(TargetError::Fatal(GdbTargetError::UnexpectedMessage))
+            }
+        }
+    }
+}
+
 impl SingleThreadResume for HyperlightSandboxTarget {
     fn resume(&mut self, _signal: Option<Signal>) -> Result<(), Self::Error> {
         log::debug!("Resume");
diff --git a/src/hyperlight_host/src/hypervisor/kvm.rs b/src/hyperlight_host/src/hypervisor/kvm.rs
index b9324334..a27936b9 100644
--- a/src/hyperlight_host/src/hypervisor/kvm.rs
+++ b/src/hyperlight_host/src/hypervisor/kvm.rs
@@ -65,6 +65,7 @@ pub(crate) fn is_hypervisor_present() -> bool {
 
 #[cfg(gdb)]
 mod debug {
+    use std::collections::HashMap;
     use std::sync::{Arc, Mutex};
 
     use hyperlight_common::mem::PAGE_SIZE;
@@ -80,6 +81,13 @@ mod debug {
     use crate::mem::layout::SandboxMemoryLayout;
     use crate::{new_error, Result};
 
+    /// Software Breakpoint size in memory
+    pub const SW_BP_SIZE: usize = 1;
+    /// Software Breakpoint opcode
+    const SW_BP_OP: u8 = 0xCC;
+    /// Software Breakpoint written to memory
+    pub const SW_BP: [u8; SW_BP_SIZE] = [SW_BP_OP];
+
     /// KVM Debug struct
     /// This struct is used to abstract the internal details of the kvm
     /// guest debugging settings
@@ -90,6 +98,8 @@ mod debug {
 
         /// Array of addresses for HW breakpoints
         hw_breakpoints: Vec<u64>,
+        /// Saves the bytes modified to enable SW breakpoints
+        sw_breakpoints: HashMap<u64, [u8; SW_BP_SIZE]>,
 
         /// Sent to KVM for enabling guest debug
         pub dbg_cfg: kvm_guest_debug,
@@ -107,6 +117,7 @@ mod debug {
             Self {
                 single_step: false,
                 hw_breakpoints: vec![],
+                sw_breakpoints: HashMap::new(),
                 dbg_cfg: dbg,
             }
         }
@@ -363,6 +374,70 @@ mod debug {
             }
         }
 
+        fn add_sw_breakpoint(
+            &mut self,
+            addr: u64,
+            dbg_mem_access_fn: Arc<Mutex<dyn DbgMemAccessHandlerCaller>>,
+        ) -> Result<bool> {
+            let addr = {
+                let debug = self
+                    .debug
+                    .as_ref()
+                    .ok_or_else(|| new_error!("Debug is not enabled"))?;
+                let addr = self.translate_gva(addr)?;
+                if debug.sw_breakpoints.contains_key(&addr) {
+                    return Ok(true);
+                }
+
+                addr
+            };
+
+            let mut save_data = [0; SW_BP_SIZE];
+            self.read_addrs(addr, &mut save_data[..], dbg_mem_access_fn.clone())?;
+            self.write_addrs(addr, &SW_BP, dbg_mem_access_fn)?;
+
+            {
+                let debug = self
+                    .debug
+                    .as_mut()
+                    .ok_or_else(|| new_error!("Debug is not enabled"))?;
+                debug.sw_breakpoints.insert(addr, save_data);
+            }
+
+            Ok(true)
+        }
+
+        fn remove_sw_breakpoint(
+            &mut self,
+            addr: u64,
+            dbg_mem_access_fn: Arc<Mutex<dyn DbgMemAccessHandlerCaller>>,
+        ) -> Result<bool> {
+            let (ret, data) = {
+                let addr = self.translate_gva(addr)?;
+                let debug = self
+                    .debug
+                    .as_mut()
+                    .ok_or_else(|| new_error!("Debug is not enabled"))?;
+
+                if debug.sw_breakpoints.contains_key(&addr) {
+                    let save_data = debug
+                        .sw_breakpoints
+                        .remove(&addr)
+                        .expect("Expected the hashmap to contain the address");
+
+                    (true, Some(save_data))
+                } else {
+                    (false, None)
+                }
+            };
+
+            if ret {
+                self.write_addrs(addr, &data.unwrap(), dbg_mem_access_fn)?;
+            }
+
+            Ok(ret)
+        }
+
         /// Gdb expects the target to be stopped when connected.
         /// This method provides a way to set a breakpoint at the entry point
         /// it does not keep this breakpoint set after the vCPU already stopped at the address
@@ -391,6 +466,9 @@ mod debug {
 
             let ip = self.get_instruction_pointer()?;
             let gpa = self.translate_gva(ip)?;
+            if debug.sw_breakpoints.contains_key(&gpa) {
+                return Ok(VcpuStopReason::SwBp);
+            }
 
             if debug.hw_breakpoints.contains(&gpa) {
                 return Ok(VcpuStopReason::HwBp);
@@ -415,6 +493,13 @@ mod debug {
                         .expect("Add hw breakpoint error");
                     Ok(DebugResponse::AddHwBreakpoint(res))
                 }
+                DebugMsg::AddSwBreakpoint(addr) => {
+                    let res = self
+                        .add_sw_breakpoint(addr, dbg_mem_access_fn)
+                        .expect("Add sw breakpoint error");
+
+                    Ok(DebugResponse::AddSwBreakpoint(res))
+                }
                 DebugMsg::Continue => {
                     self.set_single_step(false)?;
                     Ok(DebugResponse::Continue)
@@ -450,6 +535,12 @@ mod debug {
                         .expect("Remove hw breakpoint error");
                     Ok(DebugResponse::RemoveHwBreakpoint(res))
                 }
+                DebugMsg::RemoveSwBreakpoint(addr) => {
+                    let res = self
+                        .remove_sw_breakpoint(addr, dbg_mem_access_fn)
+                        .expect("Remove sw breakpoint error");
+                    Ok(DebugResponse::RemoveSwBreakpoint(res))
+                }
                 DebugMsg::Step => {
                     self.set_single_step(true)?;
                     Ok(DebugResponse::Step)

From 07da801f12f46c916093a949020bfb705fad8a6a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Doru=20Bl=C3=A2nzeanu?= <dblnz@pm.me>
Date: Tue, 4 Feb 2025 22:42:57 +0200
Subject: [PATCH 15/18] add ci checks for the gdb feature
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Signed-off-by: Doru Blânzeanu <dblnz@pm.me>
---
 .github/workflows/dep_rust.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.github/workflows/dep_rust.yml b/.github/workflows/dep_rust.yml
index 66f0a0ff..861df312 100644
--- a/.github/workflows/dep_rust.yml
+++ b/.github/workflows/dep_rust.yml
@@ -95,6 +95,7 @@ jobs:
           # make sure certain cargo features compile
           cargo check -p hyperlight-host --features crashdump
           cargo check -p hyperlight-host --features print_debug
+          cargo check -p hyperlight-host --features gdb
 
           # without any driver (shouldn't compile)
           just test-rust-feature-compilation-fail ${{ matrix.config }}

From ea250ac78eabc950c9417d556a1b52fbb782cdfc Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Doru=20Bl=C3=A2nzeanu?= <dblnz@pm.me>
Date: Tue, 4 Feb 2025 22:50:41 +0200
Subject: [PATCH 16/18] add documentation about the gdb debugging feature
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Signed-off-by: Doru Blânzeanu <dblnz@pm.me>
---
 docs/README.md                                |   1 +
 docs/how-to-debug-a-hyperlight-guest.md       | 193 ++++++++++++++++++
 .../examples/guest-debugging/main.rs          |  82 ++++++++
 3 files changed, 276 insertions(+)
 create mode 100644 docs/how-to-debug-a-hyperlight-guest.md
 create mode 100644 src/hyperlight_host/examples/guest-debugging/main.rs

diff --git a/docs/README.md b/docs/README.md
index 1b1572dd..fff51c66 100644
--- a/docs/README.md
+++ b/docs/README.md
@@ -34,6 +34,7 @@ This project is composed internally of several internal components, depicted in
 
 * [Security guidance for developers](./security-guidance-for-developers.md)
 * [Paging Development Notes](./paging-development-notes.md)
+* [How to debug a Hyperlight guest](./how-to-debug-a-hyperlight-guest.md)
 * [How to use Flatbuffers in Hyperlight](./how-to-use-flatbuffers.md)
 * [How to make a Hyperlight release](./how-to-make-releases.md)
 * [Getting Hyperlight Metrics, Logs, and Traces](./hyperlight-metrics-logs-and-traces.md)
diff --git a/docs/how-to-debug-a-hyperlight-guest.md b/docs/how-to-debug-a-hyperlight-guest.md
new file mode 100644
index 00000000..843fc7ae
--- /dev/null
+++ b/docs/how-to-debug-a-hyperlight-guest.md
@@ -0,0 +1,193 @@
+# How to debug a Hyperlight **KVM** guest using gdb
+
+Hyperlight supports gdb debugging of a **KVM** guest running inside a Hyperlight sandbox.
+When Hyperlight is compiled with the `gdb` feature enabled, a Hyperlight KVM sandbox can be configured
+to start listening for a gdb connection.
+
+## Supported features
+
+The Hyperlight `gdb` feature enables **KVM** guest debugging:
+   - an entry point breakpoint is automatically set for the guest to stop
+   - add and remove HW breakpoints (maximum 4 set breakpoints at a time)
+   - add and remove SW breakpoints
+   - read and write registers
+   - read and write addresses
+   - step/continue
+   - get code offset from target
+
+## Expected behavior
+
+Below is a list describing some cases of expected behavior from a gdb debug 
+session of a guest binary running inside a KVM Hyperlight sandbox.
+
+- when the `gdb` feature is enabled and a SandboxConfiguration is provided a
+  debug port, the created sandbox will wait for a gdb client to connect on the
+  configured port
+- when the gdb client attaches, the guest vCPU is expected to be stopped at the
+  entry point
+- if a gdb client disconnects unexpectedly, the debug session will be closed and
+  the guest will continue executing disregarding any prior breakpoints
+- if multiple sandbox instances are created, each instance will have its own
+  gdb thread listening on the configured port
+- if two sandbox instances are created with the same debug port, the second
+  instance logs an error and the gdb thread will not be created, but the sandbox
+  will continue to run without gdb debugging
+
+## Example
+
+### Sandbox configuration
+
+The `guest-debugging` example in Hyperlight demonstrates how to configure a Hyperlight
+sandbox to listen for a gdb client on a specific port.
+
+### CLI Gdb configuration
+
+One can use a gdb config file to provide the symbols and desired configuration.
+
+The below contents of the `.gdbinit` file can be used to provide a basic configuration
+to gdb startup.
+
+```gdb
+# Path to symbols
+file path/to/symbols.elf
+# The port on which Hyperlight listens for a connection
+target remote :8080
+set disassembly-flavor intel
+set disassemble-next-line on
+enable pretty-printer
+layout src
+```
+One can find more information about the `.gdbinit` file at [gdbinit(5)](https://www.man7.org/linux/man-pages/man5/gdbinit.5.html).
+
+### End to end example
+
+Using the example mentioned at [Sandbox configuration](#sandbox-configuration) 
+one can run the below commands to debug the guest binary:
+
+```bash
+# Terminal 1
+$ cargo run --example guest-debugging --features gdb
+```
+
+```bash
+# Terminal 2
+$ cat .gdbinit
+file src/tests/rust_guests/bin/debug/simpleguest
+target remote :8080
+set disassembly-flavor intel
+set disassemble-next-line on
+enable pretty-printer
+layout src
+
+$ gdb
+```
+
+### Using VSCode to debug a Hyperlight guest
+
+To replicate the above behavior using VSCode follow the below steps:
+- install the `gdb` package on the host machine
+- install the `C/C++` extension in VSCode to add debugging capabilities
+- create a `.vscode/launch.json` file in the project directory with the below content:
+    ```json
+    {
+        "version": "0.2.0",
+        "configurations": [
+            {
+                "name": "GDB",
+                "type": "cppdbg",
+                "request": "launch",
+                "program": "${workspaceFolder}/src/tests/rust_guests/bin/debug/simpleguest",
+                "args": [],
+                "stopAtEntry": true,
+                "hardwareBreakpoints": {"require": false, "limit": 4},
+                "cwd": "${workspaceFolder}",
+                "environment": [],
+                "externalConsole": false,
+                "MIMode": "gdb",
+                "miDebuggerPath": "/usr/bin/gdb",
+                "miDebuggerServerAddress": "localhost:8080",
+                "setupCommands": [
+                    {
+                        "description": "Enable pretty-printing for gdb",
+                        "text": "-enable-pretty-printing",
+                        "ignoreFailures": true
+                    },
+                    {
+                        "description": "Set Disassembly Flavor to Intel",
+                        "text": "-gdb-set disassembly-flavor intel",
+                        "ignoreFailures": true
+                    }
+                ]
+            }
+        ]
+    }
+    ```
+- in `Run and Debug` tab, select the `GDB` configuration and click on the `Run`
+  button to start the debugging session.
+  The gdb client will connect to the Hyperlight sandbox and the guest vCPU will
+  stop at the entry point.
+
+
+## How it works
+
+The gdb feature is designed to work like a Request - Response protocol between
+a thread that accepts commands from a gdb client and the hypervisor handler over
+a communication channel.
+
+All the functionality is implemented on the hypervisor side so it has access to
+the shared memory and the vCPU.
+
+The gdb thread uses the `gdbstub` crate to handle the communication with the gdb client.
+When the gdb client requests one of the supported features mentioned above, a request
+is sent over the communication channel to the hypervisor handler for the sandbox
+to resolve.
+
+Below is a sequence diagram that shows the interaction between the entities
+involved in the gdb debugging of a Hyperlight guest running inside a KVM sandbox.
+
+```
+                               ┌───────────────────────────────────────────────────────────────────────────────────────────────┐
+                               │                                       Hyperlight Sandbox                                      │
+     USER                      │                                                                                               │
+┌────────────┐                 │  ┌──────────────┐                      ┌───────────────────────────┐              ┌────────┐  │
+│ gdb client │                 │  │  gdb thread  │                      │ hypervisor handler thread │              │  vCPU  │  │
+└────────────┘                 │  └──────────────┘                      └───────────────────────────┘              └────────┘  │
+      |                        │          |               create_gdb_thread           |                                 |      │
+      |                        │          |◄─────────────────────────────────────────┌─┐         vcpu stopped          ┌─┐     │
+      |    attach              │         ┌─┐                                         │ │◄──────────────────────────────┴─┘     │
+     ┌─┐───────────────────────┼────────►│ │                                         │ │     entrypoint breakpoint      |      │
+     │ │   attach response     │         │ │                                         │ │                                |      │
+     │ │◄──────────────────────┼─────────│ │                                         │ │                                |      │
+     │ │                       │         │ │                                         │ │                                |      │
+     │ │  add_breakpoint       │         │ │                                         │ │                                |      │
+     │ │───────────────────────┼────────►│ │          add_breakpoint                 │ │                                |      │
+     │ │                       │         │ │────────────────────────────────────────►│ │  add_breakpoint                |      │
+     │ │                       │         │ │                                         │ │────┐                           |      │
+     │ │                       │         │ │                                         │ │    │                           |      │
+     │ │                       │         │ │                                         │ │◄───┘                           |      │
+     │ │                       │         │ │          add_breakpoint response        │ │                                |      │
+     │ │  add_breakpoint response        │ │◄────────────────────────────────────────│ │                                |      │
+     │ │◄──────────────────────┬─────────│ │                                         │ │                                |      │
+     │ │   continue            │         │ │                                         │ │                                |      │
+     │ │───────────────────────┼────────►│ │            continue                     │ │                                |      │
+     │ │                       │         │ │────────────────────────────────────────►│ │         resume vcpu            |      │
+     │ │                       │         │ │                                         │ │──────────────────────────────►┌─┐     │
+     │ │                       │         │ │                                         │ │                               │ │     │
+     │ │                       │         │ │                                         │ │                               │ │     │
+     │ │                       │         │ │                                         │ │                               │ │     │
+     │ │                       │         │ │                                         │ │                               │ │     │
+     │ │                       │         │ │                                         │ │         vcpu stopped          │ │     │
+     │ │                       │         │ │        notify vcpu stop reason          │ │◄──────────────────────────────┴─┘     │
+     │ │   notify vcpu stop reason       │ │◄────────────────────────────────────────│ │                                |      │
+     │ │◄──────────────────────┬─────────│ │                                         │ │                                |      │
+     │ │    continue until end │         │ │                                         │ │                                |      │
+     │ │───────────────────────┼────────►│ │            continue                     │ │         resume vcpu            |      │
+     │ │                       │         │ │────────────────────────────────────────►│ │──────────────────────────────►┌─┐     │
+     │ │                       │         │ │                                         │ │                               │ │     │
+     │ │                       │         │ │        comm channel disconnected        │ │         vcpu halted           │ │     │
+     │ │   target finished exec│         │ │◄────────────────────────────────────────┤ │◄──────────────────────────────┴─┘     │
+     │ │◄──────────────────────┼─────────┴─┘          target finished exec           └─┘                                |      │
+     │ │                       │          |                                           |                                 |      │
+     └─┘                       │          |                                           |                                 |      │
+      |                        └───────────────────────────────────────────────────────────────────────────────────────────────┘
+```
diff --git a/src/hyperlight_host/examples/guest-debugging/main.rs b/src/hyperlight_host/examples/guest-debugging/main.rs
new file mode 100644
index 00000000..409cc826
--- /dev/null
+++ b/src/hyperlight_host/examples/guest-debugging/main.rs
@@ -0,0 +1,82 @@
+/*
+Copyright 2024 The Hyperlight Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+use std::sync::{Arc, Mutex};
+use std::thread;
+
+use hyperlight_common::flatbuffer_wrappers::function_types::{ParameterValue, ReturnType};
+use hyperlight_host::func::HostFunction0;
+#[cfg(gdb)]
+use hyperlight_host::sandbox::config::DebugInfo;
+use hyperlight_host::sandbox::SandboxConfiguration;
+use hyperlight_host::sandbox_state::sandbox::EvolvableSandbox;
+use hyperlight_host::sandbox_state::transition::Noop;
+use hyperlight_host::{MultiUseSandbox, UninitializedSandbox};
+
+/// Build a sandbox configuration that enables GDB debugging when the `gdb` feature is enabled.
+fn get_sandbox_cfg() -> Option<SandboxConfiguration> {
+    #[cfg(gdb)]
+    {
+        let mut cfg = SandboxConfiguration::default();
+        let debug_info = DebugInfo { port: 8080 };
+        cfg.set_guest_debug_info(debug_info);
+
+        Some(cfg)
+    }
+
+    #[cfg(not(gdb))]
+    None
+}
+
+fn main() -> hyperlight_host::Result<()> {
+    let cfg = get_sandbox_cfg();
+
+    // Create an uninitialized sandbox with a guest binary
+    let mut uninitialized_sandbox = UninitializedSandbox::new(
+        hyperlight_host::GuestBinary::FilePath(
+            hyperlight_testing::simple_guest_as_string().unwrap(),
+        ),
+        cfg,  // sandbox configuration
+        None, // default run options
+        None, // default host print function
+    )?;
+
+    // Register a host functions
+    fn sleep_5_secs() -> hyperlight_host::Result<()> {
+        thread::sleep(std::time::Duration::from_secs(5));
+        Ok(())
+    }
+
+    let host_function = Arc::new(Mutex::new(sleep_5_secs));
+
+    host_function.register(&mut uninitialized_sandbox, "Sleep5Secs")?;
+    // Note: This function is unused, it's just here for demonstration purposes
+
+    // Initialize sandbox to be able to call host functions
+    let mut multi_use_sandbox: MultiUseSandbox = uninitialized_sandbox.evolve(Noop::default())?;
+
+    // Call guest function
+    let message = "Hello, World! I am executing inside of a VM :)\n".to_string();
+    let result = multi_use_sandbox.call_guest_function_by_name(
+        "PrintOutput", // function must be defined in the guest binary
+        ReturnType::Int,
+        Some(vec![ParameterValue::String(message.clone())]),
+    );
+
+    assert!(result.is_ok());
+
+    Ok(())
+}

From 459b1c982a291d3528737a07f5c6f0e7dcc36494 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Doru=20Bl=C3=A2nzeanu?= <dblnz@pm.me>
Date: Thu, 13 Feb 2025 17:03:14 +0200
Subject: [PATCH 17/18] add a way to report a non-fatal error to gdb
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

- sometimes gdb tries to read from an address where the vcpu translate
  method fails, so we need to handle that in such a way that the
  hypervisor handler doesn't crash

Signed-off-by: Doru Blânzeanu <dblnz@pm.me>
---
 src/hyperlight_host/src/error.rs              |  5 +++
 src/hyperlight_host/src/hypervisor/gdb/mod.rs |  1 +
 .../src/hypervisor/gdb/x86_64_target.rs       | 44 +++++++++++++++++++
 src/hyperlight_host/src/hypervisor/kvm.rs     | 31 +++++++------
 4 files changed, 68 insertions(+), 13 deletions(-)

diff --git a/src/hyperlight_host/src/error.rs b/src/hyperlight_host/src/error.rs
index e846ee12..76fdb8d0 100644
--- a/src/hyperlight_host/src/error.rs
+++ b/src/hyperlight_host/src/error.rs
@@ -275,6 +275,11 @@ pub enum HyperlightError {
     #[error("SystemTimeError {0:?}")]
     SystemTimeError(#[from] SystemTimeError),
 
+    /// Error occurred when translating guest address
+    #[error("An error occurred when translating guest address: {0:?}")]
+    #[cfg(gdb)]
+    TranslateGuestAddress(u64),
+
     /// Error occurred converting a slice to an array
     #[error("TryFromSliceError {0:?}")]
     TryFromSliceError(#[from] TryFromSliceError),
diff --git a/src/hyperlight_host/src/hypervisor/gdb/mod.rs b/src/hyperlight_host/src/hypervisor/gdb/mod.rs
index b18186a8..5ae48318 100644
--- a/src/hyperlight_host/src/hypervisor/gdb/mod.rs
+++ b/src/hyperlight_host/src/hypervisor/gdb/mod.rs
@@ -120,6 +120,7 @@ pub enum DebugResponse {
     AddSwBreakpoint(bool),
     Continue,
     DisableDebug,
+    ErrorOccurred,
     GetCodeSectionOffset(u64),
     ReadAddr(Vec<u8>),
     ReadRegisters(X86_64Regs),
diff --git a/src/hyperlight_host/src/hypervisor/gdb/x86_64_target.rs b/src/hyperlight_host/src/hypervisor/gdb/x86_64_target.rs
index 470e5993..13d3965d 100644
--- a/src/hyperlight_host/src/hypervisor/gdb/x86_64_target.rs
+++ b/src/hyperlight_host/src/hypervisor/gdb/x86_64_target.rs
@@ -87,6 +87,10 @@ impl HyperlightSandboxTarget {
 
         match self.send_command(DebugMsg::DisableDebug)? {
             DebugResponse::DisableDebug => Ok(()),
+            DebugResponse::ErrorOccurred => {
+                log::error!("Error occurred");
+                Err(GdbTargetError::UnexpectedError)
+            }
             msg => {
                 log::error!("Unexpected message received: {:?}", msg);
                 Err(GdbTargetError::UnexpectedMessage)
@@ -129,6 +133,10 @@ impl SingleThreadBase for HyperlightSandboxTarget {
 
                 Ok(v.len())
             }
+            DebugResponse::ErrorOccurred => {
+                log::error!("Error occurred");
+                Err(TargetError::NonFatal)
+            }
             msg => {
                 log::error!("Unexpected message received: {:?}", msg);
                 Err(TargetError::Fatal(GdbTargetError::UnexpectedMessage))
@@ -146,6 +154,10 @@ impl SingleThreadBase for HyperlightSandboxTarget {
 
         match self.send_command(DebugMsg::WriteAddr(gva, v))? {
             DebugResponse::WriteAddr => Ok(()),
+            DebugResponse::ErrorOccurred => {
+                log::error!("Error occurred");
+                Err(TargetError::NonFatal)
+            }
             msg => {
                 log::error!("Unexpected message received: {:?}", msg);
                 Err(TargetError::Fatal(GdbTargetError::UnexpectedMessage))
@@ -182,6 +194,10 @@ impl SingleThreadBase for HyperlightSandboxTarget {
 
                 Ok(())
             }
+            DebugResponse::ErrorOccurred => {
+                log::error!("Error occurred");
+                Err(TargetError::NonFatal)
+            }
 
             msg => {
                 log::error!("Unexpected message received: {:?}", msg);
@@ -219,6 +235,10 @@ impl SingleThreadBase for HyperlightSandboxTarget {
 
         match self.send_command(DebugMsg::WriteRegisters(regs))? {
             DebugResponse::WriteRegisters => Ok(()),
+            DebugResponse::ErrorOccurred => {
+                log::error!("Error occurred");
+                Err(TargetError::NonFatal)
+            }
             msg => {
                 log::error!("Unexpected message received: {:?}", msg);
                 Err(TargetError::Fatal(GdbTargetError::UnexpectedMessage))
@@ -240,6 +260,10 @@ impl SectionOffsets for HyperlightSandboxTarget {
                 text_seg: text,
                 data_seg: None,
             }),
+            DebugResponse::ErrorOccurred => {
+                log::error!("Error occurred");
+                Err(GdbTargetError::UnexpectedError)
+            }
             msg => {
                 log::error!("Unexpected message received: {:?}", msg);
                 Err(GdbTargetError::UnexpectedMessage)
@@ -267,6 +291,10 @@ impl HwBreakpoint for HyperlightSandboxTarget {
 
         match self.send_command(DebugMsg::AddHwBreakpoint(addr))? {
             DebugResponse::AddHwBreakpoint(rsp) => Ok(rsp),
+            DebugResponse::ErrorOccurred => {
+                log::error!("Error occurred");
+                Err(TargetError::NonFatal)
+            }
             msg => {
                 log::error!("Unexpected message received: {:?}", msg);
                 Err(TargetError::Fatal(GdbTargetError::UnexpectedMessage))
@@ -283,6 +311,10 @@ impl HwBreakpoint for HyperlightSandboxTarget {
 
         match self.send_command(DebugMsg::RemoveHwBreakpoint(addr))? {
             DebugResponse::RemoveHwBreakpoint(rsp) => Ok(rsp),
+            DebugResponse::ErrorOccurred => {
+                log::error!("Error occurred");
+                Err(TargetError::NonFatal)
+            }
             msg => {
                 log::error!("Unexpected message received: {:?}", msg);
                 Err(TargetError::Fatal(GdbTargetError::UnexpectedMessage))
@@ -301,6 +333,10 @@ impl SwBreakpoint for HyperlightSandboxTarget {
 
         match self.send_command(DebugMsg::AddSwBreakpoint(addr))? {
             DebugResponse::AddSwBreakpoint(rsp) => Ok(rsp),
+            DebugResponse::ErrorOccurred => {
+                log::error!("Error occurred");
+                Err(TargetError::NonFatal)
+            }
             msg => {
                 log::error!("Unexpected message received: {:?}", msg);
                 Err(TargetError::Fatal(GdbTargetError::UnexpectedMessage))
@@ -317,6 +353,10 @@ impl SwBreakpoint for HyperlightSandboxTarget {
 
         match self.send_command(DebugMsg::RemoveSwBreakpoint(addr))? {
             DebugResponse::RemoveSwBreakpoint(rsp) => Ok(rsp),
+            DebugResponse::ErrorOccurred => {
+                log::error!("Error occurred");
+                Err(TargetError::NonFatal)
+            }
             msg => {
                 log::error!("Unexpected message received: {:?}", msg);
                 Err(TargetError::Fatal(GdbTargetError::UnexpectedMessage))
@@ -342,6 +382,10 @@ impl SingleThreadSingleStep for HyperlightSandboxTarget {
         log::debug!("Step");
         match self.send_command(DebugMsg::Step)? {
             DebugResponse::Step => Ok(()),
+            DebugResponse::ErrorOccurred => {
+                log::error!("Error occurred");
+                Err(GdbTargetError::UnexpectedError)
+            }
             msg => {
                 log::error!("Unexpected message received: {:?}", msg);
                 Err(GdbTargetError::UnexpectedMessage)
diff --git a/src/hyperlight_host/src/hypervisor/kvm.rs b/src/hyperlight_host/src/hypervisor/kvm.rs
index a27936b9..d3cf5cc8 100644
--- a/src/hyperlight_host/src/hypervisor/kvm.rs
+++ b/src/hyperlight_host/src/hypervisor/kvm.rs
@@ -39,6 +39,8 @@ use crate::mem::memory_region::{MemoryRegion, MemoryRegionFlags};
 use crate::mem::ptr::{GuestPtr, RawPtr};
 #[cfg(gdb)]
 use crate::sandbox::config::DebugInfo;
+#[cfg(gdb)]
+use crate::HyperlightError;
 use crate::{log_then_return, new_error, Result};
 
 /// Return `true` if the KVM API is available, version 12, and has UserMemory capability, or `false` otherwise
@@ -79,7 +81,7 @@ mod debug {
     use crate::hypervisor::gdb::{DebugMsg, DebugResponse, VcpuStopReason, X86_64Regs};
     use crate::hypervisor::handlers::DbgMemAccessHandlerCaller;
     use crate::mem::layout::SandboxMemoryLayout;
-    use crate::{new_error, Result};
+    use crate::{new_error, HyperlightError, Result};
 
     /// Software Breakpoint size in memory
     pub const SW_BP_SIZE: usize = 1;
@@ -217,19 +219,13 @@ mod debug {
 
         /// Translates the guest address to physical address
         fn translate_gva(&self, gva: u64) -> Result<u64> {
-            let tr = self.vcpu_fd.translate_gva(gva).map_err(|e| {
-                new_error!(
-                    "Could not translate guest virtual address {:X}: {:?}",
-                    gva,
-                    e
-                )
-            })?;
+            let tr = self
+                .vcpu_fd
+                .translate_gva(gva)
+                .map_err(|_| HyperlightError::TranslateGuestAddress(gva))?;
 
             if tr.valid == 0 {
-                Err(new_error!(
-                    "Could not translate guest virtual address {:X}",
-                    gva
-                ))
+                Err(HyperlightError::TranslateGuestAddress(gva))
             } else {
                 Ok(tr.physical_address)
             }
@@ -919,7 +915,16 @@ impl Hypervisor for KVMDriver {
             // Wait for a message from gdb
             let req = self.recv_dbg_msg()?;
 
-            let response = self.process_dbg_request(req, dbg_mem_access_fn.clone())?;
+            let result = self.process_dbg_request(req, dbg_mem_access_fn.clone());
+
+            let response = match result {
+                Ok(response) => response,
+                // Treat non fatal errors separately so the guest doesn't fail
+                Err(HyperlightError::TranslateGuestAddress(_)) => DebugResponse::ErrorOccurred,
+                Err(e) => {
+                    return Err(e);
+                }
+            };
 
             // If the command was either step or continue, we need to run the vcpu
             let cont = matches!(

From baa46d89e8e6015713cb1c80033d64aed31d75fe Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Doru=20Bl=C3=A2nzeanu?= <dblnz@pm.me>
Date: Thu, 13 Feb 2025 19:37:30 +0200
Subject: [PATCH 18/18] fix typo from untouched file
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Signed-off-by: Doru Blânzeanu <dblnz@pm.me>
---
 src/hyperlight_host/benches/benchmarks.rs | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/hyperlight_host/benches/benchmarks.rs b/src/hyperlight_host/benches/benchmarks.rs
index 6fcd6859..db71d8a9 100644
--- a/src/hyperlight_host/benches/benchmarks.rs
+++ b/src/hyperlight_host/benches/benchmarks.rs
@@ -104,13 +104,13 @@ fn guest_call_benchmark(c: &mut Criterion) {
 fn sandbox_benchmark(c: &mut Criterion) {
     let mut group = c.benchmark_group("sandboxes");
 
-    // Benchmarks the time to create a new uninintialized sandbox.
+    // Benchmarks the time to create a new uninitialized sandbox.
     // Does **not** include the time to drop the sandbox.
     group.bench_function("create_uninitialized_sandbox", |b| {
         b.iter_with_large_drop(create_uninit_sandbox);
     });
 
-    // Benchmarks the time to create a new uninintialized sandbox and drop it.
+    // Benchmarks the time to create a new uninitialized sandbox and drop it.
     group.bench_function("create_uninitialized_sandbox_and_drop", |b| {
         b.iter(create_uninit_sandbox);
     });