x86: support hooking SYSCALL/SYSENTER instructions. we no longer share the SYSCALL callback with interrupt instructions

Author: aquynh

bindings/python/sample_x86.py

@@ -408,15 +408,15 @@ def test_x86_64_syscall():
         # write machine code to be emulated to memory
         mu.mem_write(ADDRESS, X86_CODE64_SYSCALL)
 
-        def hook_intr(mu, intno, user_data):
+        def hook_syscall(mu, user_data):
             rax = mu.reg_read(X86_REG_RAX)
-            if intno == 80 and rax == 0x100:
+            if rax == 0x100:
                 mu.reg_write(X86_REG_RAX, 0x200)
             else:
                 print('ERROR: was not expecting rax=%d in syscall' % rax)
 
         # hook interrupts for syscall
-        mu.hook_add(UC_HOOK_INTR, hook_intr)
+        mu.hook_add(UC_HOOK_INSN, hook_syscall, None, X86_INS_SYSCALL)
 
         # syscall handler is expecting rax=0x100
         mu.reg_write(X86_REG_RAX, 0x100)

bindings/python/unicorn/__init__.py

@@ -231,6 +231,7 @@ def _setup_prototype(lib, fname, restype, *argtypes):
         ctypes.c_int, ctypes.c_void_p)
 UC_HOOK_INSN_OUT_CB = ctypes.CFUNCTYPE(None, ctypes.c_size_t, ctypes.c_uint32, \
         ctypes.c_int, ctypes.c_uint32, ctypes.c_void_p)
+UC_HOOK_INSN_SYSCALL_CB = ctypes.CFUNCTYPE(None, ctypes.c_size_t, ctypes.c_void_p)
 
 
 # access to error code via @errno of UcError
@@ -383,6 +384,12 @@ def _hook_insn_out_cb(self, handle, port, size, value, user_data):
         cb(self, port, size, value, data)
 
 
+    def _hook_insn_syscall_cb(self, handle, user_data):
+        # call user's callback with self object
+        (cb, data) = self._callbacks[user_data]
+        cb(self, data)
+
+
     # add a hook
     def hook_add(self, htype, callback, user_data=None, arg1=1, arg2=0):
         _h2 = ctypes.c_size_t()
@@ -413,6 +420,8 @@ def hook_add(self, htype, callback, user_data=None, arg1=1, arg2=0):
                 cb = ctypes.cast(UC_HOOK_INSN_IN_CB(self._hook_insn_in_cb), UC_HOOK_INSN_IN_CB)
             if arg1 == x86_const.X86_INS_OUT: # OUT instruction
                 cb = ctypes.cast(UC_HOOK_INSN_OUT_CB(self._hook_insn_out_cb), UC_HOOK_INSN_OUT_CB)
+            if arg1 in (x86_const.X86_INS_SYSCALL, x86_const.X86_INS_SYSENTER): # SYSCALL/SYSENTER instruction
+                cb = ctypes.cast(UC_HOOK_INSN_SYSCALL_CB(self._hook_insn_syscall_cb), UC_HOOK_INSN_SYSCALL_CB)
             status = _uc.uc_hook_add(self._uch, ctypes.byref(_h2), htype, \
                     cb, ctypes.cast(self._callback_count, ctypes.c_void_p), insn)
         elif htype == UC_HOOK_INTR:

include/uc_priv.h

@@ -146,6 +146,7 @@ struct uc_struct {
     int hook_intr_idx; // for handling interrupt
     int hook_out_idx; // for handling OUT instruction (X86)
     int hook_in_idx; // for handling IN instruction (X86)
+    int hook_syscall_idx; // for handling SYSCALL/SYSENTER (X86)
 
 
     bool init_tcg;      // already initialized local TCGv variables?

include/unicorn/unicorn.h

@@ -19,6 +19,9 @@ extern "C" {
 
 #include "platform.h"
 
+// Handle to use with all APIs
+typedef size_t uch;
+
 #include "m68k.h"
 #include "x86.h"
 #include "arm.h"
@@ -65,9 +68,6 @@ extern "C" {
 // 1 milisecond = 1000 nanoseconds
 #define UC_MILISECOND_SCALE 1000
 
-// Handle using with all API
-typedef size_t uch;
-
 // Architecture type
 typedef enum uc_arch {
     UC_ARCH_ARM = 1,    // ARM architecture (including Thumb, Thumb-2)

include/unicorn/x86.h

@@ -8,6 +8,10 @@
 extern "C" {
 #endif
 
+// Callback function for tracing SYSCALL/SYSENTER (for uc_hook_intr())
+// @user_data: user data passed to tracing APIs.
+typedef void (*uc_cb_insn_syscall_t)(uch handle, void *user_data);
+
 //> X86 registers
 typedef enum x86_reg {
 	X86_REG_INVALID = 0,

qemu/target-i386/seg_helper.c

@@ -945,15 +945,15 @@ void helper_syscall(CPUX86State *env, int next_eip_addend)
 #else
 void helper_syscall(CPUX86State *env, int next_eip_addend)
 {
-    // Unicorn: call interrupt callback if registered
-    struct uc_struct *uc = env->uc;
-    if (uc->hook_intr_idx) {
-        ((uc_cb_hookintr_t)uc->hook_callbacks[uc->hook_intr_idx].callback)(
-            (uch)uc, 80,
-            uc->hook_callbacks[uc->hook_intr_idx].user_data);
-        env->eip += next_eip_addend;
-        return;
-    }
+	// Unicorn: call interrupt callback if registered
+	struct uc_struct *uc = env->uc;
+	if (uc->hook_syscall_idx) {
+		((uc_cb_insn_syscall_t)uc->hook_callbacks[uc->hook_syscall_idx].callback)(
+			(uch)uc, uc->hook_callbacks[uc->hook_syscall_idx].user_data);
+		env->eip += next_eip_addend;
+	}
+
+	return;
 
     int selector;
 

uc.c

@@ -645,6 +645,18 @@ static uc_err _hook_insn(struct uc_struct *uc, unsigned int insn_id, void *callb
                                   return UC_ERR_OK;
                               } else
                                   return UC_ERR_OOM;
+                     case X86_INS_SYSCALL:
+                     case X86_INS_SYSENTER:
+                              // FIXME: only one event handler at the same time
+                              i = hook_find_new(uc);
+                              if (i) {
+                                  uc->hook_callbacks[i].callback = callback;
+                                  uc->hook_callbacks[i].user_data = user_data;
+                                  *evh = i;
+                                  uc->hook_syscall_idx = i;
+                                  return UC_ERR_OK;
+                              } else
+                                  return UC_ERR_OOM;
                  }
                  break;
     }