mmu: Pseudo-recursive page tables

x86_64/lib/kernel.ld

@@ -12,7 +12,6 @@ SECTIONS {
 	 * start of the kernel for critical structures.
 	 */
 	. = ${LOAD-ADDRESS};
-
 	PROVIDE(text = .);
 	.text : AT (ADDR(.text) - KZERO) {
 		*(.text.boot)

x86_64/src/cpu.rs

@@ -54,8 +54,8 @@ pub(crate) unsafe fn ltr(selector: u16) {
 }
 
 /// Loads the "Global Table Descriptor Register" (`GDTR`) with
-/// the base address and inclusive limit inclusive of a "Global
-/// Descriptor Table" (GDT).
+/// the base address and inclusive limit of a "Global Descriptor
+/// Table" (GDT).
 ///
 /// # Safety
 /// The referred GDT must be architecturally valid.
@@ -139,7 +139,8 @@ pub(crate) unsafe fn wrmsr(msr: u32, value: u64) {
 ///
 /// # Safety
 /// The caller must ensure that the given value makes sense as
-/// a %gs segment base value.
+/// a %gs segment base value.  Note that we assume we can use
+/// the `WRGSBASE` instruction.
 pub(crate) unsafe fn wrgsbase(value: u64) {
     unsafe {
         asm!("wrgsbase {}", in(reg) value, options(att_syntax));

x86_64/src/dat.rs

@@ -7,6 +7,21 @@ use zerocopy::FromZeros;
 pub const UREG_TRAPNO_OFFSET: usize = 19 * core::mem::size_of::<u64>();
 pub const UREG_CS_OFFSET: usize = 22 * core::mem::size_of::<u64>();
 
+/// A Host Physical Address (HPA).
+///
+/// An HPA represents a physical address in the host physical
+/// address space (as distinct from a device or guest physical
+/// address space).
+#[derive(Clone, Copy, Debug, FromZeros)]
+#[repr(transparent)]
+pub struct HPA(pub u64);
+
+impl HPA {
+    pub fn from_phys(pa: u64) -> HPA {
+        HPA(pa)
+    }
+}
+
 /// The user register and trap frame structure.
 ///
 /// This stores both user state during system calls, and
@@ -89,11 +104,36 @@ impl Default for Label {
 
 /// The machine structure, which describes a CPU.
 ///
+/// The small stacks, for specific exceptions, come first and
+/// occupy the first 64KiB of the Mach.  These have guard pages
+/// immediately _after_ their data areas, which serve as guards
+/// for the subsequent data.  Since the Mach is mapped in a
+/// per-CPU portion of virtual address space that is offset from
+/// a 2MiB boundary, we know that the first stack is de facto
+/// guarded.
+///
+/// The main kstack for the scheduler comes next, and occupies
+/// the next 64KiB.  It is guarded by the NMI stack's guard page.
+///
+/// Thus, stacks and their guards take up the first 128KiB of
+/// virtual space in the Mach.
+///
+/// Next, we have the smaller Mach data itself; this includes the
+/// space required for the structures used in the pseudo-recursive
+/// page table implementation.
+///
 /// Warning: the layout of this structure is known to assembly
 /// language.
 #[derive(FromZeros)]
 #[repr(C, align(65536))]
 pub struct Mach {
+    pub debug_stack: ExStack, // Stack for debug exceptions
+    pub bp_stack: ExStack,    // Stack for breakpoint exceptions
+    pub df_stack: ExStack,    // Stack for double faults
+    pub nmi_stack: ExStack,   // Stack for NMIs
+    pub zero: Page,           // Mapped to (per-node) read-only zeroed page
+    pub stack: KStack,        // Kernel stack for scheduler
+
     me: *mut Mach,            // %gs:0 is a `*mut Mach` pointing to this `Mach`.
     scratch: usize,           // A scratch word used on entry to kernel
     splpc: usize,             // PC of last caller to ` k`.  Cleared by `spllo`.
@@ -114,24 +154,22 @@ pub struct Mach {
 
     sched: Label,
 
-    // Architecturally defined.
-    pub tss: Tss,
-
-    // All preceding data fits within a single 4KiB page.  Structures
-    // that follow are sized in page multiples and aligned.
-    pml4: Page,               // PML4 root page table for this Mach
-    pml3: Page,               // The PML3 that maps the kernel for this mach
-    pml2: Page,               // PML2 for low 1GiB
-    pml1: Page,               // PML1 for low 2MiB
-    pub idt: Idt,             // Interrupt descriptor table
-    zero: Page,               // Read-only, zeroed page
-    pub df_stack: ExStack,    // Stack for double faults
-    pub debug_stack: ExStack, // Stack for debug exceptions
-    pub nmi_stack: ExStack,   // Stack for NMIs
-    pub stack: KStack,        // Kernel stack for scheduler
-    pub gdt: Gdt,             // Gdt is aligned to 64KiB.
+    // All preceding data after the stack fits within a single 4KiB page.
+    // Paging structures that follow are sized in page multiples and aligned.
+    pml4: PTable,  // PML4 root page table for this Mach
+    pml3: PTable,  // PML3 for rec 512GiB (root of all subtrees)
+    pml2: PTable,  // PML2 for rec 1GiB
+    pml1: PTable,  // PML1 for rec 2MiB (PML4 and all PML3s)
+    mpml3: PTable, // PML3 for mapping region
+    mpml2: PTable, // PML2 for mapping region
+    mpml1: PTable, // PML1 for mapping region
+
+    // Architecturally defined data.
+    pub tss: Tss, // Truly per-CPU
+    idt: Idt,     // Mapped to per-node IDT
+    gdt: Gdt,     // Mapped to per-CPU GDT; padded to 64k with zeros
 }
-static_assertions::const_assert_eq!(core::mem::offset_of!(Mach, pml4), 4096);
+static_assertions::const_assert_eq!(core::mem::offset_of!(Mach, pml4), 65536 * 2 + 4096);
 static_assertions::const_assert_eq!(core::mem::offset_of!(Mach, stack), 65536);
 
 impl Mach {
@@ -142,10 +180,9 @@ impl Mach {
             (0, &mut self.stack),
             (trap::NMI_TRAPNO, &mut self.nmi_stack),
             (trap::DEBUG_TRAPNO, &mut self.debug_stack),
+            (trap::BREAKPOINT_TRAPNO, &mut self.bp_stack),
             (trap::DOUBLE_FAULT_TRAPNO, &mut self.df_stack),
         ]);
-        self.gdt.init(&self.tss);
-        self.idt.init(trap::stubs());
         unsafe {
             self.gdt.load();
             self.idt.load();
@@ -224,6 +261,16 @@ impl Flags {
     }
 }
 
+#[derive(FromZeros)]
+#[repr(C, align(4096))]
+pub struct PTable([u64; 512]);
+
+impl PTable {
+    pub fn array_mut(&mut self) -> &mut [u64; 512] {
+        &mut self.0
+    }
+}
+
 /// The smallest basic page type.
 #[derive(FromZeros)]
 #[repr(C, align(4096))]

x86_64/src/l.S

@@ -39,11 +39,12 @@
 .set KCPUZERO,			0xffffff0000000000
 .set KMACH,			(1*MiB)
 .set KTOFF,			(2*MiB)
+.set KLOSTK,			0x8000
 
-.set MACHSTKSZ,			(16*PGSZ)
-.set MACHSTKOFF,		(16*PGSZ)
-.set MACHGDTSZ,			(16*PGSZ)
-.set MACHGDTOFF,		(32*PGSZ)
+.set MACHSTKSZ,			(64*KiB)
+.set MACHSTKOFF,		(64*KiB)
+.set MACHGDTSZ,			(64*KiB)
+.set MACHGDTOFF,		(192*KiB)
 
 .set Cr0PE,			(1<<0)		// Protected Mode Enable
 .set Cr0MP,			(1<<1)		// Monitor Coprocessor
@@ -56,6 +57,7 @@
 .set Cr4PSE,			(1<<4)		// Page-Size Extensions
 .set Cr4PAE,			(1<<5)		// Physical Address Extension
 .set Cr4PGE,			(1<<7)		// Page-Global Enable
+.set Cr4FSGSBASE,		(1<<16)		// (RD|WR)(FS|GS)BASE Extension
 
 .set IA32_EFER,			0xc0000080	// Extended Feature Enable
 
@@ -156,7 +158,7 @@ start:
 	// off the page size extension; it is always eanbled in PAE mode.
 	movl	%cr4, %eax
 	andl	$~Cr4PSE, %eax
-	orl	$(Cr4PGE|Cr4PAE), %eax
+	orl	$(Cr4PGE|Cr4PAE|Cr4FSGSBASE), %eax
 	movl	%eax, %cr4
 
 	// Set up the Extended Feature Enable (EFER) MSR, enabling
@@ -203,19 +205,26 @@ start:
 	movw	%ax, %gs
 
 	// We can now use linked addresses for the stack and code.
-	// We'll jump into the kernel from here.
-	movabsq	$(KCPUZERO+KMACH), %rdi
-	leaq	MACHSTKOFF+MACHSTKSZ(%rdi), %rsp
+	// Jump to the code that initialized ccNUMA node 0 and the
+	// cpu0 Mach, returning to warp64, which removes the
+	// identity map, sets the SP to the scheduler stack, and
+	// jumps
+	leaq	KMACH, %rdi
+	leaq	KLOSTK, %rsp
 	movabsq	$warp64, %rax
 	pushq	%rax
+	movabsq	$init0, %rax
+	pushq	%rax
 	ret
 
 warp64:
+	// Now load the stack pointer with our Mach stack.
+	movabsq	$(KCPUZERO+KMACH), %rdi
+	leaq	MACHSTKOFF+MACHSTKSZ(%rdi), %rsp
 	// At this point, we are fully in the kernel virtual
 	// address space and can discard the identity mapping.
 	// To do so, we load the early PML4 that does not have
 	// that entry.
-	movq	$(earlypml4 - KZERO), %rax
 	movq	%rax, %cr3			// Also flushes TLB.
 
 	// A pointer to the Mach is already in %rdi, and is the
@@ -369,7 +378,9 @@ b1982:
 	// Enable and activate Long Mode.
 	movl	%cr4, %eax
 	andl	$~Cr4PSE, %eax		// PSE always true in long mode
-	orl	$(Cr4PGE|Cr4PAE), %eax	// Page Global, Phys. Address Ext
+	// Enable page global, physical address extension, and FS/GS base
+	// load/store instruction support.
+	orl	$(Cr4PGE|Cr4PAE|Cr4FSGSBASE), %eax
 	movl	%eax, %cr4
 
 	movl	$IA32_EFER, %ecx	// Extended Feature Enable
@@ -386,28 +397,27 @@ b1982:
 
 .code64
 1:
-	leaq	APMACH, %rdi
-	movq	%rdi, %rsp
-	movabsq	$apwarp64, %rax
-	pushq	%rax
-	ret
-	ud2
-
-apwarp64:
 	movabsq	$gdtdescv, %rax
 	lgdt	(%rax)
 	pushq	$GdtCODE64
 	pushq	$(1f-KZERO)
 	lretq
 1:
-
 	xorl	%eax, %eax
 	movw	%ax, %ds
 	movw	%ax, %es
 	movw	%ax, %fs
 	movw	%ax, %gs
 	movw	%ax, %ss
 
+	leaq	APMACH, %rdi
+	movq	%rdi, %rsp
+	movabsq	$apwarp64, %rax
+	pushq	%rax
+	ret
+	ud2
+
+apwarp64:
 	// Move to the real PML4 from our Mach.
 	movq	(%rdi), %rax
 	movq	%rax, %cr3
@@ -419,7 +429,7 @@ apwarp64:
 
 	pushq	$0
 	xorl	%ebp, %ebp
-	movq	8(%rdi), %rax			// m->scratch
+	movq	128*KiB+8(%rdi), %rax			// m->scratch
 	pushq	%rax
 	pushq	$2				// Clear flags
 	popfq
@@ -432,13 +442,8 @@ e1978:
 bootpml4:
 	.quad	bootidentitypt3 - KZERO + (PteRW | PteP)
 	.space	4096 - 3*8
-	.quad	earlymachpt3 - KZERO + (PteRW | PteP)
-	.quad	earlykernpt3 - KZERO + (PteRW | PteP)
-
-earlypml4:
-	.space 4096 - 2*8
-	.quad	earlymachpt3 - KZERO + (PteRW | PteP)
-	.quad	earlykernpt3 - KZERO + (PteRW | PteP)
+	.quad	bootmachpt3 - KZERO + (PteRW | PteP)
+	.quad	bootkernpt3 - KZERO + (PteRW | PteP)
 
 bootidentitypt3:
 	.quad	(0<<30) + (PtePS | PteRW | PteP)
@@ -447,11 +452,15 @@ bootidentitypt3:
 	.quad	(3<<30) + (PtePS | PteRW | PteP)
 	.space	4096 - 4*8
 
-earlymachpt3:
+bootmachpt3:
+	// The CPU part
+	.quad	(0<<30) + (PtePS | PteRW | PteP)
+	.space	4096/2 - 1*8
+	// The node part
 	.quad	(0<<30) + (PtePS | PteRW | PteP)
-	.space	4096 - 1*8
+	.space	4096/2 - 1*8
 
-earlykernpt3:
+bootkernpt3:
 	.space	4096 - 2*8
 	.quad	(0<<30) + (PtePS | PteRW | PteP)
 	.quad	0

x86_64/src/main.rs

@@ -10,6 +10,7 @@ mod allocator;
 mod cpu;
 mod dat;
 mod devcons;
+mod node0;
 mod pio;
 mod proc;
 mod syscall;