Automatically generate structure member offsets for assembly

.gitignore

@@ -10,6 +10,8 @@ Makeconf.local
 .DS_Store
 *.swp
 *.xz
+asm-offsets.s
+asm-offsets.h
 grub/boot/grub
 drivers/acpi/acpica/source/
 third-party/libpfm/COPYING

Makefile

@@ -144,9 +144,15 @@ ACPICA_INSTALL := $(shell [ -d $(ACPICA_DEST_DIR)/source ] ||
                           $(HARDLINK) $(ACPICA_DEST_DIR)/acktf.h $(ACPICA_INCLUDE)/platform/acktf.h)
 endif
 
-SOURCES     := $(shell find . -name \*.c)
+ASM_OFFSETS_C := asm-offsets.c
+ASM_OFFSETS_S := asm-offsets.s
+ASM_OFFSETS_H := asm-offsets.h
+ASM_OFFSETS_SH := $(KTF_ROOT)/tools/asm-offsets/asm-offsets.sh
+
+SOURCES     := $(shell find . -name \*.c -and -not -name $(ASM_OFFSETS_C))
 HEADERS     := $(shell find . -name \*.h)
 ASM_SOURCES := $(shell find . -name \*.S)
+ASM_OFFSETS := $(shell find . -name $(ASM_OFFSETS_C))
 LINK_SCRIPT := $(shell find . -name \*.ld)
 
 SYMBOLS_NAME := symbols
@@ -198,14 +204,19 @@ $(PFMLIB_ARCHIVE): $(PFMLIB_TARBALL)
 	$(VERBOSE) cp $(PFMLIB_DIR)/lib/$(PFMLIB_NAME).a $(PFMLIB_DIR)/
 	$(VERBOSE) find $(PFMLIB_DIR) -name \*.c -delete
 
-%.o: %.S
+%.o: %.S $(ASM_OFFSETS_H)
 	@echo "AS " $@
 	$(VERBOSE) $(CC) -c -o $@ $(AFLAGS) $<
 
 %.o: %.c $(PFMLIB_ARCHIVE)
 	@echo "CC " $@
 	$(VERBOSE) $(CC) -c -o $@ $(CFLAGS) $<
 
+$(ASM_OFFSETS_H): $(ASM_OFFSETS)
+	@echo "GEN" include/$(shell dirname $<)/$(ASM_OFFSETS_H)
+	$(VERBOSE) $(CC) $(CFLAGS) -S -g0 -o $(KTF_ROOT)/$(shell dirname $<)/$(ASM_OFFSETS_S) $<
+	$(VERBOSE) $(ASM_OFFSETS_SH) $(KTF_ROOT)/$(shell dirname $<)/$(ASM_OFFSETS_S) $(KTF_ROOT)/include/$(shell dirname $<)/$(ASM_OFFSETS_H)
+
 DEPFILES := $(OBJS:.o=.d)
 -include $(wildcard $(DEPFILES))
 
@@ -219,6 +230,8 @@ clean:
 	$(VERBOSE) find $(KTF_ROOT) -name \*.img -delete
 	$(VERBOSE) find $(KTF_ROOT) -name \*.xz -delete
 	$(VERBOSE) find $(KTF_ROOT) -name cscope.\* -delete
+	$(VERBOSE) find $(KTF_ROOT) -name $(ASM_OFFSETS_S) -delete
+	$(VERBOSE) find $(KTF_ROOT) -name $(ASM_OFFSETS_H) -delete
 	$(VERBOSE) find $(PFMLIB_DIR) -mindepth 1 ! -name $(PFMLIB_NAME)-$(PFMLIB_VER).tar.gz -delete
 	$(VERBOSE) $(RM) -rf $(ACPICA_DEST_DIR)/source
 	$(VERBOSE) $(RM) -f $(TARGET_DEBUG)

arch/x86/asm-offsets.c

@@ -0,0 +1,70 @@
+/*
+ * Copyright © 2023 Open Source Security, Inc.
+ * All Rights Reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * 1. Redistributions of source code must retain the above copyright notice, this
+ *    list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright notice,
+ *    this list of conditions and the following disclaimer in the documentation
+ *    and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
+ * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include <compiler.h>
+#include <percpu.h>
+
+#define EMIT_DEFINE(s, v)                                                                \
+    asm volatile(".ascii \"@@#define " #s " %0 /* " #v " */@@\"\n" ::"i"(v))
+#define OFFSETOF(__symbol, __type, __member)                                             \
+    EMIT_DEFINE(__symbol, offsetof(__type, __member));
+
+void __asm_offset_header(void) {
+    OFFSETOF(usermode_private, percpu_t, usermode_private);
+
+    OFFSETOF(cpu_exc_error_code, cpu_exc_t, error_code);
+    OFFSETOF(cpu_exc_vector, cpu_exc_t, vector);
+    OFFSETOF(cpu_exc_ip, cpu_exc_t, _ASM_IP);
+    OFFSETOF(cpu_exc_cs, cpu_exc_t, cs);
+    OFFSETOF(cpu_exc_flags, cpu_exc_t, _ASM_FLAGS);
+    OFFSETOF(cpu_exc_sp, cpu_exc_t, _ASM_SP);
+    OFFSETOF(cpu_exc_ss, cpu_exc_t, ss);
+
+    OFFSETOF(cpu_regs_ax, cpu_regs_t, _ASM_AX);
+    OFFSETOF(cpu_regs_bx, cpu_regs_t, _ASM_BX);
+    OFFSETOF(cpu_regs_cx, cpu_regs_t, _ASM_CX);
+    OFFSETOF(cpu_regs_dx, cpu_regs_t, _ASM_DX);
+    OFFSETOF(cpu_regs_si, cpu_regs_t, _ASM_SI);
+    OFFSETOF(cpu_regs_di, cpu_regs_t, _ASM_DI);
+    OFFSETOF(cpu_regs_bp, cpu_regs_t, _ASM_BP);
+#if defined(__x86_64__)
+    OFFSETOF(cpu_regs_r8, cpu_regs_t, r8);
+    OFFSETOF(cpu_regs_r9, cpu_regs_t, r9);
+    OFFSETOF(cpu_regs_r10, cpu_regs_t, r10);
+    OFFSETOF(cpu_regs_r11, cpu_regs_t, r11);
+    OFFSETOF(cpu_regs_r12, cpu_regs_t, r12);
+    OFFSETOF(cpu_regs_r13, cpu_regs_t, r13);
+    OFFSETOF(cpu_regs_r14, cpu_regs_t, r14);
+    OFFSETOF(cpu_regs_r15, cpu_regs_t, r15);
+#endif
+
+    OFFSETOF(cpu_regs_error_code, cpu_regs_t, exc.error_code);
+    OFFSETOF(cpu_regs_vector, cpu_regs_t, exc.vector);
+    OFFSETOF(cpu_regs_ip, cpu_regs_t, exc._ASM_IP);
+    OFFSETOF(cpu_regs_cs, cpu_regs_t, exc.cs);
+    OFFSETOF(cpu_regs_flags, cpu_regs_t, exc._ASM_FLAGS);
+    OFFSETOF(cpu_regs_sp, cpu_regs_t, exc._ASM_SP);
+    OFFSETOF(cpu_regs_ss, cpu_regs_t, exc.ss);
+}

arch/x86/entry.S

@@ -23,51 +23,52 @@
  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
  * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  */
+#include <asm-offsets.h>
 #include <asm-macros.h>
 #include <processor.h>
 #include <segment.h>
 #include <page.h>
 #include <traps.h>
 
 .macro _handle_usermode cr3
-    testb $0x3, 8(%_ASM_SP)
+    testb $0x3, cpu_exc_cs(%_ASM_SP)
     jz 1f /* skip if from kernel mode */
         SET_CR3 \cr3
         swapgs
     1:
 .endm
 
-.macro from_usermode
+.macro enter_from_usermode
     _handle_usermode cr3
 .endm
 
-.macro to_usermode
+.macro enter_to_usermode
     _handle_usermode user_cr3
 .endm
 
 .macro exception_handler sym vec has_error_code
 ENTRY(entry_\sym)
-    from_usermode
+    enter_from_usermode
 
     .if \has_error_code == 0
         push $0
     .endif
 
-    movl $\vec, 0x4(%_ASM_SP)
+    movl $\vec, cpu_exc_vector(%_ASM_SP)
     jmp handle_exception
 END_FUNC(entry_\sym)
 .endm
 
 .macro interrupt_handler sym func
 ENTRY(asm_interrupt_handler_\sym)
-    from_usermode
+    enter_from_usermode
 
     cld
     SAVE_ALL_REGS
     call \func
     RESTORE_ALL_REGS
 
-    to_usermode
+    enter_to_usermode
     IRET
 END_FUNC(asm_interrupt_handler_\sym)
 .endm
@@ -110,7 +111,7 @@ ENTRY(handle_exception)
     add $4, %_ASM_SP
 #endif
 
-    to_usermode
+    enter_to_usermode
     IRET
 END_FUNC(handle_exception)
 
@@ -129,25 +130,25 @@ ENTRY(enter_usermode)
     PUSHF
 
     /* Save stack pointer onto per-cpu */
-    mov %_ASM_SP, %gs:(%_ASM_DX)
+    mov %_ASM_SP, %gs:usermode_private
 
     /* Move to user stack */
-    mov %_ASM_CX, %_ASM_SP
+    mov %_ASM_DX, %_ASM_SP
 
     /* SS + SP */
     push $__USER_DS
-    push %_ASM_CX
+    push %_ASM_DX
 
     /* EFLAGS */
     PUSHF
 
     orl $X86_EFLAGS_IOPL, (%_ASM_SP)
 
     /* CS + IP */
-    pushq $__USER_CS
+    push $__USER_CS
     push $usermode_stub
 
-    to_usermode
+    enter_to_usermode
     IRET
 END_FUNC(enter_usermode)
 

arch/x86/traps.c

@@ -181,7 +181,7 @@ static void dump_general_regs(const struct cpu_regs *regs) {
            "\nRIP=0x%016lx\n\n",
            regs->_ASM_AX, regs->r8, regs->_ASM_BX, regs->r9, regs->_ASM_CX, regs->r10,
            regs->_ASM_DX, regs->r11, regs->_ASM_SI, regs->r12, regs->_ASM_DI, regs->r13,
-           regs->_ASM_BP, regs->r14, regs->_ASM_SP, regs->r15, regs->_ASM_IP);
+           regs->_ASM_BP, regs->r14, regs->exc._ASM_SP, regs->r15, regs->exc._ASM_IP);
 }
 
 static void dump_control_regs(const struct cpu_regs *regs) {
@@ -197,17 +197,17 @@ static void dump_segment_regs(const struct cpu_regs *regs) {
            "ES=0x%04lx FS=0x%04lx GS=0x%04lx\n"
            "EXCEPTION:\n"
            "CS=0x%04x SS=0x%04x\n\n",
-           read_cs(), read_ds(), read_ss(), read_es(), read_fs(), read_gs(), regs->cs,
-           regs->ss);
+           read_cs(), read_ds(), read_ss(), read_es(), read_fs(), read_gs(), regs->exc.cs,
+           regs->exc.ss);
 }
 
 static void dump_flags(const struct cpu_regs *regs) {
-    printk("RFLAGS=0x%016lx\n\n", regs->_ASM_FLAGS);
+    printk("RFLAGS=0x%016lx\n\n", regs->exc._ASM_FLAGS);
 }
 
 #define DUMP_STACK_LINES 32
 static void dump_stack(const struct cpu_regs *regs, unsigned words) {
-    unsigned long *sp = (unsigned long *) regs->_ASM_SP;
+    unsigned long *sp = (unsigned long *) regs->exc._ASM_SP;
     unsigned lines = DUMP_STACK_LINES;
 
     printk("STACK[%p]:", sp);
@@ -282,11 +282,11 @@ void print_callstack(const void *sp, const void *ip) {
 
 static bool extables_fixup(struct cpu_regs *regs) {
     for (extable_entry_t *cur = __start_extables; cur < __stop_extables; ++cur) {
-        if (regs->_ASM_IP != cur->fault_addr)
+        if (regs->exc._ASM_IP != cur->fault_addr)
             continue;
 
         if (cur->fixup)
-            regs->_ASM_IP = cur->fixup;
+            regs->exc._ASM_IP = cur->fixup;
         else if (cur->cb)
             cur->cb(regs);
 
@@ -299,22 +299,22 @@ static bool extables_fixup(struct cpu_regs *regs) {
 void do_exception(struct cpu_regs *regs) {
     static char ec_str[32], panic_str[128];
 
-    if (!from_usermode(regs->cs) && extables_fixup(regs))
+    if (!enter_from_usermode(regs->exc.cs) && extables_fixup(regs))
         return;
 
     dump_regs(regs);
-    print_callstack(_ptr(regs->_ASM_SP), _ptr(regs->_ASM_IP));
+    print_callstack(_ptr(regs->exc._ASM_SP), _ptr(regs->exc._ASM_IP));
 
-    if (has_error_code(regs->vector))
-        x86_ex_decode_error_code(ec_str, sizeof(ec_str), regs->vector, regs->error_code);
+    if (has_error_code(regs->exc.vector))
+        x86_ex_decode_error_code(ec_str, sizeof(ec_str), regs->exc.vector, regs->exc.error_code);
 
     snprintf(panic_str, sizeof(panic_str),
              "#%s %sat IP: 0x%02x:0x%016lx SP: 0x%02x:0x%016lx\n",
-             exception_names[regs->vector], ec_str, regs->cs, regs->_ASM_IP, regs->ss,
-             regs->_ASM_SP);
+             exception_names[regs->exc.vector], ec_str, regs->exc.cs, regs->exc._ASM_IP, regs->exc.ss,
+             regs->exc._ASM_SP);
 
     /* Handle user tasks' exceptions */
-    if (from_usermode(regs->cs)) {
+    if (enter_from_usermode(regs->exc.cs)) {
         printk("Task exception: %s\n", panic_str);
         goto_syscall_exit(-EFAULT);
     }

common/sched.c

@@ -204,8 +204,7 @@ static void run_task(task_t *task) {
 
     set_task_state(task, TASK_STATE_RUNNING);
     if (task->type == TASK_TYPE_USER)
-        task->result = enter_usermode(task->func, task->arg,
-                                      PERCPU_OFFSET(usermode_private), task->stack);
+        task->result = enter_usermode(task->func, task->arg, task->stack);
     else
         task->result = task->func(task->arg);
     set_task_state(task, TASK_STATE_DONE);

include/arch/x86/processor.h

@@ -241,6 +241,20 @@ typedef uint64_t x86_reg_t;
 typedef uint32_t x86_reg_t;
 #endif
 
+struct cpu_exc {
+    x86_ex_error_code_t error_code;
+    /* Populated by exception entry */
+    uint32_t vector;
+
+    /* Hardware exception */
+    x86_reg_t _ASM_IP;
+    uint16_t cs, _pad_cs[3];
+    x86_reg_t _ASM_FLAGS;
+    x86_reg_t _ASM_SP;
+    uint16_t ss, _pad_ss[3];
+} __packed;
+typedef struct cpu_exc cpu_exc_t;
+
 struct cpu_regs {
     x86_reg_t r15;
     x86_reg_t r14;
@@ -258,17 +272,8 @@ struct cpu_regs {
     x86_reg_t _ASM_BX;
     x86_reg_t _ASM_AX;
 
-    x86_ex_error_code_t error_code;
-    /* Populated by exception entry */
-    uint32_t vector;
-
-    /* Hardware exception */
-    x86_reg_t _ASM_IP;
-    uint16_t cs, _pad_cs[3];
-    x86_reg_t _ASM_FLAGS;
-    x86_reg_t _ASM_SP;
-    uint16_t ss, _pad_ss[3];
-};
+    cpu_exc_t exc;
+} __packed;
 typedef struct cpu_regs cpu_regs_t;
 
 union msr_star {

include/usermode.h

@@ -36,8 +36,8 @@
 
 /* Static declarations */
 
-static inline bool from_usermode(uint16_t cs) {
-    return (cs & 0x3) != 0;
+static inline bool enter_from_usermode(uint16_t cs) {
+    return (cs & 0x3) == 0x3;
 }
 
 static inline void goto_syscall_exit(long exit_code) {
@@ -47,8 +47,7 @@ static inline void goto_syscall_exit(long exit_code) {
 
 /* External declarations */
 
-extern unsigned long enter_usermode(task_func_t fn, void *fn_arg,
-                                    unsigned long usermode_private, void *user_stack);
+extern unsigned long enter_usermode(task_func_t fn, void *fn_arg, void *user_stack);
 extern void __naked syscall_handler(void);
 
 extern void init_usermode(percpu_t *percpu);

tools/asm-offsets/asm-offsets.sh

@@ -0,0 +1,15 @@
+#!/bin/bash
+
+INPUT=$1
+OUTPUT=$2
+
+cat <<EOT > "$OUTPUT"
+/* This file has been automatically generated from $INPUT. */
+
+#ifndef KTF_ASM_OFFSETS_H
+#define KTF_ASM_OFFSETS_H
+
+$(awk -F '@@' '/\.ascii "@@/ {print $2}' "$INPUT" | tr -d '$')
+
+#endif /* KTF_ASM_OFFSETS_H */
+EOT