mmu: Pseudo-recursive page tables

This prototypes support for self-referential mappings by making
use of pseudo-recursive page tables.  These are "standard" page
tables, but we allocate some portion of the address space they
define to what we call "the recursive region".  We then map the
page tables that make up the non-recursive regions of the
address space into the recursive region, using addresses derived
from the addresses those tables map.

These maps are set up early in boot, before entry to main, when
setting up the mapping for the per-node data and per-CPU Mach
structure for ccNUMA node 0 and CPU 0, respectively.

Signed-off-by: Dan Cross <cross@gajendra.net>

Author: Dan Cross

x86_64/lib/kernel.ld

@@ -12,7 +12,6 @@ SECTIONS {
 	 * start of the kernel for critical structures.
 	 */
 	. = ${LOAD-ADDRESS};
-
 	PROVIDE(text = .);
 	.text : AT (ADDR(.text) - KZERO) {
 		*(.text.boot)

x86_64/src/cpu.rs

@@ -54,8 +54,8 @@ pub(crate) unsafe fn ltr(selector: u16) {
 }
 
 /// Loads the "Global Table Descriptor Register" (`GDTR`) with
-/// the base address and inclusive limit inclusive of a "Global
-/// Descriptor Table" (GDT).
+/// the base address and inclusive limit of a "Global Descriptor
+/// Table" (GDT).
 ///
 /// # Safety
 /// The referred GDT must be architecturally valid.
@@ -139,7 +139,8 @@ pub(crate) unsafe fn wrmsr(msr: u32, value: u64) {
 ///
 /// # Safety
 /// The caller must ensure that the given value makes sense as
-/// a %gs segment base value.
+/// a %gs segment base value.  Note that we assume we can use
+/// the `WRGSBASE` instruction.
 pub(crate) unsafe fn wrgsbase(value: u64) {
     unsafe {
         asm!("wrgsbase {}", in(reg) value, options(att_syntax));

x86_64/src/dat.rs

@@ -7,6 +7,16 @@ use zerocopy::FromZeros;
 pub const UREG_TRAPNO_OFFSET: usize = 19 * core::mem::size_of::<u64>();
 pub const UREG_CS_OFFSET: usize = 22 * core::mem::size_of::<u64>();
 
+#[derive(Clone, Copy, Debug, FromZeros)]
+#[repr(transparent)]
+pub struct HPA(pub u64);
+
+impl HPA {
+    pub fn from_phys(pa: u64) -> HPA {
+        HPA(pa)
+    }
+}
+
 /// The user register and trap frame structure.
 ///
 /// This stores both user state during system calls, and
@@ -89,11 +99,36 @@ impl Default for Label {
 
 /// The machine structure, which describes a CPU.
 ///
+/// The small stacks, for specific exceptions, come first and
+/// occupy the first 64KiB of the Mach.  These have guard pages
+/// immediately _after_ their data areas, which serve as guards
+/// for the subsequent data.  Since the Mach is mapped in a
+/// per-CPU portion of virtual address space that is offset from
+/// a 2MiB boundary, we know that the first stack is de facto
+/// guarded.
+///
+/// The main kstack for the scheduler comes next, and occupies
+/// the next 64KiB.  It is guarded by the NMI stack's guard page.
+///
+/// Thus, stacks and their guards take up the first 128KiB of
+/// virtual space in the Mach.
+///
+/// Next, we have the smaller Mach data itself; this includes the
+/// space required for the structures used in the pseudo-recursive
+/// page table implementation.
+///
 /// Warning: the layout of this structure is known to assembly
 /// language.
 #[derive(FromZeros)]
 #[repr(C, align(65536))]
 pub struct Mach {
+    pub debug_stack: ExStack, // Stack for debug exceptions
+    pub bp_stack: ExStack,    // Stack for breakpoint exceptions
+    pub df_stack: ExStack,    // Stack for double faults
+    pub nmi_stack: ExStack,   // Stack for NMIs
+    pub zero: Page,           // Mapped to (per-node) read-only zeroed page
+    pub stack: KStack,        // Kernel stack for scheduler
+
     me: *mut Mach,            // %gs:0 is a `*mut Mach` pointing to this `Mach`.
     scratch: usize,           // A scratch word used on entry to kernel
     splpc: usize,             // PC of last caller to ` k`.  Cleared by `spllo`.
@@ -114,24 +149,22 @@ pub struct Mach {
 
     sched: Label,
 
-    // Architecturally defined.
-    pub tss: Tss,
-
-    // All preceding data fits within a single 4KiB page.  Structures
-    // that follow are sized in page multiples and aligned.
-    pml4: Page,               // PML4 root page table for this Mach
-    pml3: Page,               // The PML3 that maps the kernel for this mach
-    pml2: Page,               // PML2 for low 1GiB
-    pml1: Page,               // PML1 for low 2MiB
-    pub idt: Idt,             // Interrupt descriptor table
-    zero: Page,               // Read-only, zeroed page
-    pub df_stack: ExStack,    // Stack for double faults
-    pub debug_stack: ExStack, // Stack for debug exceptions
-    pub nmi_stack: ExStack,   // Stack for NMIs
-    pub stack: KStack,        // Kernel stack for scheduler
-    pub gdt: Gdt,             // Gdt is aligned to 64KiB.
+    // All preceding data after the stack fits within a single 4KiB page.
+    // Paging structures that follow are sized in page multiples and aligned.
+    pml4: PTable,  // PML4 root page table for this Mach
+    pml3: PTable,  // PML3 for rec 512GiB (root of all subtrees)
+    pml2: PTable,  // PML2 for rec 1GiB
+    pml1: PTable,  // PML1 for rec 2MiB (PML4 and all PML3s)
+    mpml3: PTable, // PML3 for mapping region
+    mpml2: PTable, // PML2 for mapping region
+    mpml1: PTable, // PML1 for mapping region
+
+    // Architecturally defined data.
+    pub tss: Tss, // Truly per-CPU
+    idt: Idt,     // Mapped to per-node IDT
+    gdt: Gdt,     // Mapped to per-CPU GDT; padded to 64k with zeros
 }
-static_assertions::const_assert_eq!(core::mem::offset_of!(Mach, pml4), 4096);
+static_assertions::const_assert_eq!(core::mem::offset_of!(Mach, pml4), 65536 * 2 + 4096);
 static_assertions::const_assert_eq!(core::mem::offset_of!(Mach, stack), 65536);
 
 impl Mach {
@@ -142,10 +175,9 @@ impl Mach {
             (0, &mut self.stack),
             (trap::NMI_TRAPNO, &mut self.nmi_stack),
             (trap::DEBUG_TRAPNO, &mut self.debug_stack),
+            (trap::BREAKPOINT_TRAPNO, &mut self.bp_stack),
             (trap::DOUBLE_FAULT_TRAPNO, &mut self.df_stack),
         ]);
-        self.gdt.init(&self.tss);
-        self.idt.init(trap::stubs());
         unsafe {
             self.gdt.load();
             self.idt.load();
@@ -224,6 +256,16 @@ impl Flags {
     }
 }
 
+#[derive(FromZeros)]
+#[repr(C, align(4096))]
+pub struct PTable([u64; 512]);
+
+impl PTable {
+    pub fn array_mut(&mut self) -> &mut [u64; 512] {
+        &mut self.0
+    }
+}
+
 /// The smallest basic page type.
 #[derive(FromZeros)]
 #[repr(C, align(4096))]

x86_64/src/l.S

@@ -39,11 +39,12 @@
 .set KCPUZERO,			0xffffff0000000000
 .set KMACH,			(1*MiB)
 .set KTOFF,			(2*MiB)
+.set KLOSTK,			0x8000
 
-.set MACHSTKSZ,			(16*PGSZ)
-.set MACHSTKOFF,		(16*PGSZ)
-.set MACHGDTSZ,			(16*PGSZ)
-.set MACHGDTOFF,		(32*PGSZ)
+.set MACHSTKSZ,			(64*KiB)
+.set MACHSTKOFF,		(64*KiB)
+.set MACHGDTSZ,			(64*KiB)
+.set MACHGDTOFF,		(192*KiB)
 
 .set Cr0PE,			(1<<0)		// Protected Mode Enable
 .set Cr0MP,			(1<<1)		// Monitor Coprocessor
@@ -56,6 +57,7 @@
 .set Cr4PSE,			(1<<4)		// Page-Size Extensions
 .set Cr4PAE,			(1<<5)		// Physical Address Extension
 .set Cr4PGE,			(1<<7)		// Page-Global Enable
+.set Cr4FSGSBASE,		(1<<16)		// (RD|WR)(FS|GS)BASE Extension
 
 .set IA32_EFER,			0xc0000080	// Extended Feature Enable
 
@@ -156,7 +158,7 @@ start:
 	// off the page size extension; it is always eanbled in PAE mode.
 	movl	%cr4, %eax
 	andl	$~Cr4PSE, %eax
-	orl	$(Cr4PGE|Cr4PAE), %eax
+	orl	$(Cr4PGE|Cr4PAE|Cr4FSGSBASE), %eax
 	movl	%eax, %cr4
 
 	// Set up the Extended Feature Enable (EFER) MSR, enabling
@@ -203,19 +205,26 @@ start:
 	movw	%ax, %gs
 
 	// We can now use linked addresses for the stack and code.
-	// We'll jump into the kernel from here.
-	movabsq	$(KCPUZERO+KMACH), %rdi
-	leaq	MACHSTKOFF+MACHSTKSZ(%rdi), %rsp
+	// Jump to the code that initialized ccNUMA node 0 and the
+	// cpu0 Mach, returning to warp64, which removes the
+	// identity map, sets the SP to the scheduler stack, and
+	// jumps
+	leaq	KMACH, %rdi
+	leaq	KLOSTK, %rsp
 	movabsq	$warp64, %rax
 	pushq	%rax
+	movabsq	$init0, %rax
+	pushq	%rax
 	ret
 
 warp64:
+	// Now load the stack pointer with our Mach stack.
+	movabsq	$(KCPUZERO+KMACH), %rdi
+	leaq	MACHSTKOFF+MACHSTKSZ(%rdi), %rsp
 	// At this point, we are fully in the kernel virtual
 	// address space and can discard the identity mapping.
 	// To do so, we load the early PML4 that does not have
 	// that entry.
-	movq	$(earlypml4 - KZERO), %rax
 	movq	%rax, %cr3			// Also flushes TLB.
 
 	// A pointer to the Mach is already in %rdi, and is the
@@ -369,7 +378,9 @@ b1982:
 	// Enable and activate Long Mode.
 	movl	%cr4, %eax
 	andl	$~Cr4PSE, %eax		// PSE always true in long mode
-	orl	$(Cr4PGE|Cr4PAE), %eax	// Page Global, Phys. Address Ext
+	// Enable page global, physical address extension, and FS/GS base
+	// load/store instruction support.
+	orl	$(Cr4PGE|Cr4PAE|Cr4FSGSBASE), %eax
 	movl	%eax, %cr4
 
 	movl	$IA32_EFER, %ecx	// Extended Feature Enable
@@ -386,28 +397,27 @@ b1982:
 
 .code64
 1:
-	leaq	APMACH, %rdi
-	movq	%rdi, %rsp
-	movabsq	$apwarp64, %rax
-	pushq	%rax
-	ret
-	ud2
-
-apwarp64:
 	movabsq	$gdtdescv, %rax
 	lgdt	(%rax)
 	pushq	$GdtCODE64
 	pushq	$(1f-KZERO)
 	lretq
 1:
-
 	xorl	%eax, %eax
 	movw	%ax, %ds
 	movw	%ax, %es
 	movw	%ax, %fs
 	movw	%ax, %gs
 	movw	%ax, %ss
 
+	leaq	APMACH, %rdi
+	movq	%rdi, %rsp
+	movabsq	$apwarp64, %rax
+	pushq	%rax
+	ret
+	ud2
+
+apwarp64:
 	// Move to the real PML4 from our Mach.
 	movq	(%rdi), %rax
 	movq	%rax, %cr3
@@ -419,7 +429,7 @@ apwarp64:
 
 	pushq	$0
 	xorl	%ebp, %ebp
-	movq	8(%rdi), %rax			// m->scratch
+	movq	128*KiB+8(%rdi), %rax			// m->scratch
 	pushq	%rax
 	pushq	$2				// Clear flags
 	popfq
@@ -432,13 +442,8 @@ e1978:
 bootpml4:
 	.quad	bootidentitypt3 - KZERO + (PteRW | PteP)
 	.space	4096 - 3*8
-	.quad	earlymachpt3 - KZERO + (PteRW | PteP)
-	.quad	earlykernpt3 - KZERO + (PteRW | PteP)
-
-earlypml4:
-	.space 4096 - 2*8
-	.quad	earlymachpt3 - KZERO + (PteRW | PteP)
-	.quad	earlykernpt3 - KZERO + (PteRW | PteP)
+	.quad	bootmachpt3 - KZERO + (PteRW | PteP)
+	.quad	bootkernpt3 - KZERO + (PteRW | PteP)
 
 bootidentitypt3:
 	.quad	(0<<30) + (PtePS | PteRW | PteP)
@@ -447,11 +452,15 @@ bootidentitypt3:
 	.quad	(3<<30) + (PtePS | PteRW | PteP)
 	.space	4096 - 4*8
 
-earlymachpt3:
+bootmachpt3:
+	// The CPU part
+	.quad	(0<<30) + (PtePS | PteRW | PteP)
+	.space	4096/2 - 1*8
+	// The node part
 	.quad	(0<<30) + (PtePS | PteRW | PteP)
-	.space	4096 - 1*8
+	.space	4096/2 - 1*8
 
-earlykernpt3:
+bootkernpt3:
 	.space	4096 - 2*8
 	.quad	(0<<30) + (PtePS | PteRW | PteP)
 	.quad	0

x86_64/src/main.rs

@@ -10,6 +10,7 @@ mod allocator;
 mod cpu;
 mod dat;
 mod devcons;
+mod node0;
 mod pio;
 mod proc;
 mod syscall;